diff --git a/.ansible-lint-ignore b/.ansible-lint-ignore index 91fdd99bf1..f1540d08f9 100644 --- a/.ansible-lint-ignore +++ b/.ansible-lint-ignore @@ -21,6 +21,7 @@ roles/sys-svc-mail-msmtp/tasks/roles/sys-svc-mail-smtp/tasks/disable.yml load-fa roles/web-app-bigbluebutton/tasks/roles/svc-bkp-volume-2-local/tasks/03_seed-database-to-backup.yml load-failure[filenotfounderror] roles/web-app-mailu/tasks/roles/sys-svc-mail-smtp/tasks/disable.yml load-failure[filenotfounderror] roles/web-app-matrix/tasks/flavor/compose/roles/svc-bkp-volume-2-local/tasks/03_seed-database-to-backup.yml load-failure[filenotfounderror] +roles/web-app-stalwart/tasks/roles/sys-svc-mail-smtp/tasks/disable.yml load-failure[filenotfounderror] roles/web-app-prometheus/tasks/roles/sys-svc-compose/tasks/utils/network/create.yml load-failure[filenotfounderror] roles/web-app-minio/tasks/roles/sys-svc-compose/tasks/utils/network/create.yml load-failure[filenotfounderror] roles/web-svc-seaweedfs/tasks/roles/sys-svc-compose/tasks/utils/network/create.yml load-failure[filenotfounderror] diff --git a/.claude/settings.json b/.claude/settings.json index 6b24066880..ddd06716db 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -503,7 +503,6 @@ "Bash(git send-pack*)", "Bash(git-setup-remotes*)", "Bash(git-sign-push*)", - "Bash(rm -rf*)", "Bash(sudo *)", "Bash(until *)", "Bash(while *)" @@ -569,9 +568,11 @@ "Bash(gh repo sync*)", "Bash(gh workflow run*)", "Bash(git commit*)", + "Edit(**/.claude/*.json)", "Edit(CHANGELOG.md)", "Edit(pyproject.toml)", "Skill(update-config*)", + "Write(**/.claude/*.json)", "Write(CHANGELOG.md)", "Write(pyproject.toml)" ], @@ -603,12 +604,25 @@ "*.pythonhosted.org", "*.taiga.io", "*.windows.net", + "127.0.0.1", + "127.0.0.10", + "127.0.0.11", + "127.0.0.12", + "127.0.0.2", + "127.0.0.3", + "127.0.0.4", + "127.0.0.5", + "127.0.0.6", + "127.0.0.7", + "127.0.0.8", + "127.0.0.9", "cybermaster.space", "ghcr.io", "github.com", "infinito.nexus", "infinito.test", "letsencrypt.org", + "localhost", "monogramm.io", "pypi.org", "raw.githubusercontent.com" diff --git a/.github/workflows/README.md b/.github/workflows/README.md index db46210d03..398d459d15 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -290,7 +290,8 @@ failed chunk: `skipped` counts as passed, so an empty chunk never blocks the chain. After fixing what broke a sweep, `resume_from_chunk` re-enters at that index instead of re-running the green chunks. Both are inputs on `entry-manual-steer.yml`; -the other entry points take the defaults. A retrigger carries the source run's +the other entry points take the defaults, except that a release push on `main` +(a `vX.Y.Z` tag on the pushed commit) runs with `chunk_gate: false`. A retrigger carries the source run's `chunk_gate` over unless `infinito administration deploy ci trigger --chunk-gate false` overrides it; the `i8ciallon` alias is `i8ciall` with that override, for sweeps that should report every chunk before anyone looks. diff --git a/.github/workflows/call-test-deploy.yml b/.github/workflows/call-test-deploy.yml index c256190372..160da5f607 100644 --- a/.github/workflows/call-test-deploy.yml +++ b/.github/workflows/call-test-deploy.yml @@ -139,6 +139,8 @@ env: access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} INFINITO_IMAGE_TAG: ci-${{ github.sha }} INFINITO_GHCR_MIRROR_PREFIX: mirror + INFINITO_MAIL_PRODUCTION_HOST: ${{ vars.INFINITO_MAIL_PRODUCTION_HOST }} + INFINITO_MAIL_PRODUCTION_RESOLVER: ${{ vars.INFINITO_MAIL_PRODUCTION_RESOLVER }} INFINITO_PLAYWRIGHT_KEEP: ${{ vars.PLAYWRIGHT_KEEP_ALL }} INFINITO_API_OPENAI_KEY: ${{ secrets.API_OPENAI_KEY || vars.API_OPENAI_KEY }} INFINITO_API_ANTHROPIC_KEY: ${{ secrets.API_ANTHROPIC_KEY || vars.API_ANTHROPIC_KEY }} diff --git a/.github/workflows/call-test.yml b/.github/workflows/call-test.yml index ded49e9ff2..ac26225993 100644 --- a/.github/workflows/call-test.yml +++ b/.github/workflows/call-test.yml @@ -17,6 +17,9 @@ jobs: steps: - name: ⬇️ Checkout repository uses: actions/checkout@v7.0.0 + with: + fetch-depth: 0 + fetch-tags: true - name: 🔧 Set up Python uses: actions/setup-python@v6 diff --git a/.github/workflows/entry-cancel-superseded.yml b/.github/workflows/entry-cancel-superseded.yml index c0f789cc82..dfd1df7907 100644 --- a/.github/workflows/entry-cancel-superseded.yml +++ b/.github/workflows/entry-cancel-superseded.yml @@ -26,7 +26,7 @@ permissions: jobs: cancel-branch-runs: name: 🚫 Cancel superseded branch runs - if: ${{ github.event_name == 'push' && github.ref_name != 'main' }} + if: ${{ github.event_name == 'push' && (github.ref_name != 'main' || (github.ref_name == 'main' && startsWith(github.event.head_commit.message, 'Release version '))) }} runs-on: ubuntu-latest permissions: actions: write diff --git a/.github/workflows/entry-push-latest.yml b/.github/workflows/entry-push-latest.yml index 171be265b0..ed3f8bde73 100644 --- a/.github/workflows/entry-push-latest.yml +++ b/.github/workflows/entry-push-latest.yml @@ -2,7 +2,7 @@ name: "📤 Push" concurrency: group: global-ci-${{ github.repository }}-${{ github.ref_name }} - cancel-in-progress: ${{ github.ref_name != 'main' }} + cancel-in-progress: ${{ github.ref_name != 'main' || (github.ref_name == 'main' && startsWith(github.event.head_commit.message, 'Release version ')) }} on: push: @@ -138,6 +138,7 @@ jobs: uses: ./.github/workflows/call-orchestrator.yml with: priority: ${{ needs.resolve-priority.outputs.priority }} + chunk_gate: ${{ !(github.ref_name == 'main' && needs.detect-release.outputs.is_version == 'true') }} secrets: inherit release-guard: diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ef77c0d82..cc6b7d8ff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,146 @@ # Changelog +## [15.0.0] - 2026-10-03 + +**Breaking Changes** + +* **Stalwart replaces Mailu as the default mail provider.** *MAIL_PROVIDER* now defaults to + *web-app-stalwart*, and every application switches its mail settings on only while the + configured provider is deployed. An inventory that keeps Mailu must set + *MAIL_PROVIDER: web-app-mailu*, otherwise its applications stop sending mail. Mailu is now + *deprecated*: it keeps working until it is removed, but new deployments must not adopt it, + and CI no longer deploys or tests it, the mailbox import included. To move over, deploy + Stalwart next to Mailu and switch on *services.stalwart.migration.import_mailu*: in compose + mode the role imports every inventory account's mail from Mailu over IMAP, keeping + folders, flags and dates, and re-runs skip what is already there. See + [web-app-stalwart](roles/web-app-stalwart/README.md). + +* **PostgreSQL 18 on the official image.** PGDG archived the Debian 11 repository that the + old *postgis/postgis:17-3.5* base installed from, so every build of the shared database + image failed. *svc-db-postgres* and the per-application database now run the official + multi-arch *postgres:18* image, and the data volume mounts at */var/lib/postgresql*. + Existing PostgreSQL 17 clusters are not migrated. Dump every database before you deploy + this release and restore it into the new cluster afterwards. PostGIS is installed only for + a role that declares it. + +* **Major versions arrive with the image updates.** MariaDB moves from 12.3 to 13.0 and + upgrades its system tables on start. Odoo moves from 19.0 to 20.0 and does not migrate an + existing database across major versions on its own, so back up Odoo before you deploy. + XWiki moves from 17.10 to 18.8, and the OpenLDAP base from Debian 12 to 13. + +**For Users** + +* **Stalwart mail server.** The new *web-app-stalwart* role runs SMTP, submission, IMAP, + POP3, JMAP, ManageSieve, CalDAV, CardDAV and WebDAV in one server, with spam filtering, + DKIM and DMARC signing and a web admin interface, plus Roundcube webmail and a ClamAV scan + of incoming mail. Domains, accounts, TLS, Keycloak sign-in, DKIM keys and DNS records are + provisioned through Stalwart's management API on every deploy, in compose and in swarm + mode. Two silent failures are closed: a Keycloak that was still starting could leave + Stalwart without single sign-on for the life of the process, and the server wrote no logs + at all. Both are fixed. See [web-app-stalwart](roles/web-app-stalwart/README.md). + +* **Mail to and from onion addresses.** The new *svc-net-tor-smtp* role accepts mail for + *.onion* recipients and delivers it over Tor. The applications reach an onion mail + provider too: 27 roles now resolve the provider's onion host inside their containers and + skip STARTTLS towards it, since no certificate can exist for an onion name and Tor already + encrypts the link. GitLab's test suite checks that a password reset mail actually arrives. + +* **GitLab migrations no longer run out of locks.** GitLab's database migration failed on + PostgreSQL 18 with "out of shared memory". The shared and the per-application PostgreSQL + now allow 256 locks per transaction. + +* **Backups survive a file truncated mid-copy.** *baudolo* 7.0.2 lets the stopped checksum + copy replace a live pre-copy that a writing container broke by truncating files under + rsync, which used to abort the backup. + +**For Developers** + +* **The mail provider is a service, not a role name.** Stalwart declares *provides: email* + and Mailu *covers: [email]*. The new *mail_provider* lookup returns the configured provider + while it is deployed, otherwise the first deployed role offering *email*, and reads the + cluster-wide groups so swarm workers agree with the manager. The port-collision lint + accepts a collision only inside a registry-declared provider-alternative group, and a new + lint fails every role that reads the *email* lookup without emitting + *container_extra_hosts*. + +* **Deprecated roles are untested.** A role in the *deprecated* lifecycle stage is no longer + deployed or tested by CI and cannot block a release, so Stalwart stopped pulling Mailu into + its deploy round. The allowed lifecycle values now have one source, the stage list in + [lifecycle.md](docs/contributing/design/role/services/lifecycle.md). + +* **Release pushes on main.** A push to *main* whose head commit starts with + *Release version* cancels the running *main* pipeline, so the release starts at once. A + release-tagged commit deploys every chunk instead of stopping at the first failed one, and + still ends red when any chunk fails, which keeps a broken release from being published. A + new integration test requires every release tag from v13.0.0 on to sit on that subject, + and the test job now checks out the full history with tags. + +* **Cancelling CI runs.** Pull request runs fail fast: the first failed row cancels its + siblings in the deploy and workspace matrices, while push and manual runs keep every row. + Identical deploy rows from different runs queue behind each other again, keyed on every + matrix axis. A cancel now also stops jobs that used to run after a failure: + *!cancelled()* replaces *always()*, enforced by lint, and only *done* and + *report-main-failures* still report a cancelled run. CodeRabbit reviews draft pull + requests too. + +* **Onion port probe runs everywhere.** A role with more than one onion-forwarded port wrote + an unquoted port list into *test.env*, so sourcing it left the list empty and the probe + passed without checking anything. The list is quoted now, and a unit test sources the + rendered file in compose and swarm mode. + +* **Agent sandbox.** *git commit --amend* is denied, the host's credential files stay + unreadable inside the sandbox, sandboxed commands may reach local services on 127.0.0.1 + to 127.0.0.12 and localhost, and third-party plugins are no longer switched on for every + contributor. + +* **Ansible collections from Git first.** The automated retune made Git the preferred + source for Ansible collections, with Galaxy as the fallback, after Galaxy's median install + time reached three times Git's. + +* Image and dependency version jumps (net since 14.2.0): + * *svc-db-postgres*: postgis/postgis 17-3.5 to postgres 18 + * *svc-db-mariadb*: 12.3 to 13.0 + * *svc-db-redis*: 8.10.1-alpine to 8.10.2-alpine + * *svc-db-openldap* (Debian base): 12.15-slim to 13.7-slim + * *svc-ai-ollama*: 0.34.2 to 0.35.0 + * *svc-runner* (BuildKit): v0.33.0 to v0.33.1 + * *svc-net-tor-smtp*: newly on python 3.14-slim + * *web-app-stalwart*: newly pinned to Stalwart v0.16.24, Roundcube 1.7.4-apache, ClamAV + 1.5.4 and postgres 18 + * *web-app-dashboard* (port-ui): 2.1.3 to 2.2.2 + * *web-app-erpnext*: v16.35.0 to v16.37.0 + * *web-app-espocrm*: 10.0.8 to 10.0.9 + * *web-app-gitlab*: v19.3.2 to v19.4.1 (all six CNG component images) + * *web-app-joomla*: 6.1.3-php8.3-apache to 6.1.4-php8.3-apache + * *web-app-keycloak*: 26.7.4 to 26.8.0, oauth2-proxy v7.15.4 to v7.15.5 + * *web-app-magento*: OpenSearch 3.8.0 to 3.9.0 + * *web-app-matomo*: 5.13.0 to 5.14.0 + * *web-app-matrix*: Synapse v1.160.0 to v1.162.0, Element v1.12.28 to v1.12.30 + * *web-app-odoo*: 19.0 to 20.0 + * *web-app-peertube*: v8.3.0 to v8.3.1 + * *web-app-seaweedfs*: 4.46 to 4.48 + * *web-app-shopware*: OpenSearch 3.8.0 to 3.9.0 + * *web-app-snipe-it*: v8.7.2-alpine to v8.8.0-alpine + * *web-app-socialhome*: 2026.9.18 to 2026.10.2 + * *web-app-xwiki*: 17.10.13 to 18.8.0 + * *web-app-bluesky* (view): 1.132.0 to 1.134.0 + * *web-app-discourse*: v2026.8.0 to v2026.9.0 + * *web-svc-logout*: universal-logout v1.3.1 to v1.4.1 + * *web-svc-xmpp*: 26.07 to 26.09 + * *pkgmgr*: v1.16.1 to v2.0.2 + * *backup-docker-to-local* (*baudolo*): 7.0.1 to 7.0.2 + * *ruff* (dev): 0.16.7 to 0.16.9 + * *brace-expansion* (dev): ^5.0.9 to ^5.0.12 + +**Contributors** + +* [Alejandro Roman Ibanez](https://github.com/AlejandroRomanIbanez): Stalwart as the default + mail provider in compose and swarm mode, the Tor SMTP gateway, onion mail delivery for + every mail-sending role, the Mailu import and the provider resolution +* [Kevin Veen-Birkenbach](https://veen.world): PostgreSQL 18, release-aware CI cancellation + and chunk gating, the release tag test, the Mailu deprecation and the untested deprecated + stage, the agent sandbox policy, review and version maintenance + ## [14.2.0] - 2026-09-22 **For Users** diff --git a/Makefile b/Makefile index 8389219d1c..51ac62fcd9 100644 --- a/Makefile +++ b/Makefile @@ -202,8 +202,10 @@ compose-app-restart: # Param full_cycle: true | false — when true, also run the async update pass # Param variant: matrix round index to pin the redeploy to a specific variant # Param debug: true | false (default: from default.env) +# Param skip_compile: true | false — skip the in-container entry.sh recompile (use when the venv is prebuilt) compose-deploy: @$(if $(debug),INFINITO_DEBUG="$(debug)") \ + $(if $(skip_compile),INFINITO_SKIP_COMPILE="$(skip_compile)") \ bash scripts/tests/deploy/local/deploy/main.sh .PHONY: compose-down diff --git a/README.md b/README.md index beb3a6f9aa..e3aa0b90e3 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Every invokable role in the tested lifecycle envelope, with its upstream homepag | Name | Status | Description | More | Integrated with | |---|---|---|---|---| | [Agent Broker](roles/svc-ai-agent-broker/) | beta | Per-user Hermes and OpenClaw agents started on demand behind an OpenAI-compatible endpoint | [🛠️](roles/svc-ai-agent-broker/README.md#quick-setup) | [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [OpenClaw](roles/web-app-openclaw/) | -| [Akaunting](roles/web-app-akaunting/) | beta | Self-hosted platform for online accounting, bookkeeping, and business finance management | [🌐](https://akaunting.com/) [🎬](https://www.youtube.com/watch?v=UTVe_Zob_k0) [🛠️](roles/web-app-akaunting/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Akaunting](roles/web-app-akaunting/) | beta | Self-hosted platform for online accounting, bookkeeping, and business finance management | [🌐](https://akaunting.com/) [🎬](https://www.youtube.com/watch?v=UTVe_Zob_k0) [🛠️](roles/web-app-akaunting/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [API](roles/web-svc-api/) | alpha | Public REST API serving role data, history and translations of Infinito.Nexus for every ref of core and its forks | [🌐](https://fastapi.tiangolo.com/) [🛠️](roles/web-svc-api/README.md#quick-setup) | [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [Asset Mirror (Privacy Proxy)](roles/web-svc-mirror/) | alpha | Nginx-based privacy proxy that mirrors whitelisted third-party assets under a first-party domain | [🛠️](roles/web-svc-mirror/README.md#quick-setup) | [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [Assets Server](roles/web-svc-asset/) | beta | Nginx-based server for hosting static assets copied from a source directory to the web data path | [🛠️](roles/web-svc-asset/README.md#quick-setup) | [File Server](roles/web-svc-file/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | @@ -71,12 +71,13 @@ Every invokable role in the tested lifecycle envelope, with its upstream homepag | [Backup Remote Server](roles/svc-bkp-remote-2-local/) | beta | Scheduled pull-style backup replicating remote hosts onto this host over SSH and rsync as hard-linked snapshots | [🛠️](roles/svc-bkp-remote-2-local/README.md#quick-setup) | [Backup to External Device](roles/svc-bkp-local-2-device/) | | [Backup to External Device](roles/svc-bkp-local-2-device/) | beta | Plug-triggered systemd backup for hard-link deduplicated snapshots of the local backup tree onto a swappable USB device | [🛠️](roles/svc-bkp-local-2-device/README.md#quick-setup) | | | [Base](roles/dsk-base/) | beta | Shared premise of every desktop role, the login account its files belong to | [🛠️](roles/dsk-base/README.md#quick-setup) | [Workstation User](roles/user-workstation/) | -| [Baserow](roles/web-app-baserow/) | beta | Self-hosted no-code database platform with a spreadsheet-like interface for building and collaborating on data | [🌐](https://baserow.io/) [🎬](https://www.youtube.com/watch?v=5yFMfP96_4M) [🛠️](roles/web-app-baserow/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [BigBlueButton](roles/web-app-bigbluebutton/) | beta | Self-hosted web conferencing platform for virtual classrooms, online meetings, and real-time collaboration | [🌐](https://bigbluebutton.org/) [🎬](https://www.youtube.com/watch?v=YRkQB6MJllI) [🛠️](roles/web-app-bigbluebutton/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Collabora](roles/web-svc-collabora/), [Corporate Design](roles/web-svc-css/), [Coturn](roles/web-svc-coturn/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Universal Logout](roles/web-svc-logout/) | -| [Bluesky](roles/web-app-bluesky/) | beta | Self-hosted decentralized social network on the open AT Protocol, with its own Personal Data Server | [🌐](https://bsky.social/) [🛠️](roles/web-app-bluesky/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [BookWyrm](roles/web-app-bookwyrm/) | beta | Self-hosted federated reading platform for book reviews and reading tracking across the Fediverse | [🌐](https://joinbookwyrm.com/) [🛠️](roles/web-app-bookwyrm/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Baserow](roles/web-app-baserow/) | beta | Self-hosted no-code database platform with a spreadsheet-like interface for building and collaborating on data | [🌐](https://baserow.io/) [🎬](https://www.youtube.com/watch?v=5yFMfP96_4M) [🛠️](roles/web-app-baserow/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [BigBlueButton](roles/web-app-bigbluebutton/) | beta | Self-hosted web conferencing platform for virtual classrooms, online meetings, and real-time collaboration | [🌐](https://bigbluebutton.org/) [🎬](https://www.youtube.com/watch?v=YRkQB6MJllI) [🛠️](roles/web-app-bigbluebutton/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Collabora](roles/web-svc-collabora/), [Corporate Design](roles/web-svc-css/), [Coturn](roles/web-svc-coturn/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Universal Logout](roles/web-svc-logout/) | +| [Blender](roles/dsk-blender/) | alpha | Blender workstation installation wired to the agents on the host through the Model Context Protocol | [🌐](https://www.blender.org) [🛠️](roles/dsk-blender/README.md#quick-setup) | [Base](roles/dsk-base/), [Claude Code](roles/dsk-gnt-claude/), [Code](roles/dsk-code/) | +| [Bluesky](roles/web-app-bluesky/) | beta | Self-hosted decentralized social network on the open AT Protocol, with its own Personal Data Server | [🌐](https://bsky.social/) [🛠️](roles/web-app-bluesky/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [BookWyrm](roles/web-app-bookwyrm/) | beta | Self-hosted federated reading platform for book reviews and reading tracking across the Fediverse | [🌐](https://joinbookwyrm.com/) [🛠️](roles/web-app-bookwyrm/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Bridgy Fed](roles/web-app-bridgy-fed/) | beta | Federation bridge connecting ActivityPub (Fediverse), the AT Protocol (Bluesky), and the IndieWeb | [🌐](https://fed.brid.gy/) [🛠️](roles/web-app-bridgy-fed/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | -| [Checkmk](roles/web-app-checkmk/) | beta | Open-source IT and application monitoring server behind SSO with native LDAP user management | [🌐](https://checkmk.com/) [🎬](https://www.youtube.com/watch?v=opO-SOgOJ1I) [🛠️](roles/web-app-checkmk/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Checkmk](roles/web-app-checkmk/) | beta | Open-source IT and application monitoring server behind SSO with native LDAP user management | [🌐](https://checkmk.com/) [🎬](https://www.youtube.com/watch?v=opO-SOgOJ1I) [🛠️](roles/web-app-checkmk/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Chess](roles/web-app-chess/) | beta | Federated chess server for playing and following rule-validated games over ActivityPub across the Fediverse | [🌐](https://castling.club/) [🛠️](roles/web-app-chess/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [Claude Code](roles/dsk-gnt-claude/) | alpha | Terminal coding agent from Anthropic, pointed at the platform's own model gateway instead of the vendor API | [🌐](https://code.claude.com/) [🛠️](roles/dsk-gnt-claude/README.md#quick-setup) | [Base](roles/dsk-base/), [LiteLLM Gateway](roles/svc-ai-litellm/) | | [Code](roles/dsk-code/) | alpha | Visual Studio Code workstation editor in a selectable flavor, carrying the extensions of the agents the host deploys | [🌐](https://github.com/microsoft/vscode) [🛠️](roles/dsk-code/README.md#quick-setup) | [Base](roles/dsk-base/), [Claude Code](roles/dsk-gnt-claude/), [Codex](roles/dsk-gnt-codex/) | @@ -87,26 +88,26 @@ Every invokable role in the tested lifecycle envelope, with its upstream homepag | [Coturn](roles/web-svc-coturn/) | beta | Free and open-source TURN/STUN relay server for NAT traversal in WebRTC and real-time communication | [🌐](https://github.com/coturn/coturn) [🛠️](roles/web-svc-coturn/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Prometheus](roles/web-app-prometheus/) | | [Cursor](roles/dsk-gnt-cursor/) | alpha | AI-first code editor installed on the workstation, whose endpoint override stays an operator step | [🌐](https://cursor.com/) [🛠️](roles/dsk-gnt-cursor/README.md#quick-setup) | [Base](roles/dsk-base/) | | [Dashboard](roles/web-app-dashboard/) | beta | Self-hosted portal for unified access to all deployed applications via a customizable tile and menu interface | [🛠️](roles/web-app-dashboard/README.md#quick-setup) | [Assets Server](roles/web-svc-asset/), [Content Delivery Network](roles/web-svc-cdn/), [Corporate Design](roles/web-svc-css/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Simple Icons](roles/web-svc-simpleicons/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Decidim](roles/web-app-decidim/) | beta | Open-source participatory democracy platform for public consultations, civic processes, and community voting | [🌐](https://decidim.org/) [🎬](https://www.youtube.com/watch?v=luSdPZ-wUIY) [🛠️](roles/web-app-decidim/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Discourse](roles/web-app-discourse/) | beta | Open-source discussion and community forum platform with moderation, notifications, and plugin support | [🌐](https://www.discourse.org/) [🎬](https://www.youtube.com/watch?v=z3qp_uB46FA) [🛠️](roles/web-app-discourse/README.md#quick-setup) | [Assets Server](roles/web-svc-asset/), [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Decidim](roles/web-app-decidim/) | beta | Open-source participatory democracy platform for public consultations, civic processes, and community voting | [🌐](https://decidim.org/) [🎬](https://www.youtube.com/watch?v=luSdPZ-wUIY) [🛠️](roles/web-app-decidim/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Discourse](roles/web-app-discourse/) | beta | Open-source discussion and community forum platform with moderation, notifications, and plugin support | [🌐](https://www.discourse.org/) [🎬](https://www.youtube.com/watch?v=z3qp_uB46FA) [🛠️](roles/web-app-discourse/README.md#quick-setup) | [Assets Server](roles/web-svc-asset/), [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [BigBlueButton](roles/web-app-bigbluebutton/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Jitsi Meet](roles/web-app-jitsi/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mastodon](roles/web-app-mastodon/), [Matomo](roles/web-app-matomo/), [Matrix](roles/web-app-matrix/), [Mattermost](roles/web-app-mattermost/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Translation Gateway](roles/web-svc-translate/), [Universal Logout](roles/web-svc-logout/) | | [Docker Registry](roles/svc-registry-docker/) | beta | Cluster-local container image registry for serving custom-built images to Docker Swarm worker nodes | [🌐](https://distribution.github.io/distribution/) [🛠️](roles/svc-registry-docker/README.md#quick-setup) | [Docker Swarm](roles/svc-swarm-node/) | | [Docker Registry Cache](roles/svc-registry-cache/) | beta | Cluster-local pull-through cache for shared Docker Hub base-image pulls, cutting duplicate transfers and rate limits | [🛠️](roles/svc-registry-cache/README.md#quick-setup) | [Docker Swarm](roles/svc-swarm-node/) | | [Docker Swarm](roles/svc-swarm-node/) | beta | Container orchestration cluster for scheduling services across pooled manager and worker nodes | [🛠️](roles/svc-swarm-node/README.md#quick-setup) | | | [Docker Swarm Manager](roles/svc-swarm-manager/) | beta | Marker role tagging a host as the Docker Swarm manager node, with the init logic delegated to svc-swarm-node | [🛠️](roles/svc-swarm-manager/README.md#quick-setup) | [Docker Swarm](roles/svc-swarm-node/) | | [Documentation](roles/web-app-docs/) | beta | Infinito.Nexus documentation for the latest commit and every release, built on demand with Sphinx | [🌐](https://www.sphinx-doc.org/) [🛠️](roles/web-app-docs/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [Elasticsearch](roles/svc-db-elasticsearch/) | beta | Central search and analytics engine with per-consumer index namespaces isolating tenants in a shared Docker deployment | [🌐](https://www.elastic.co/elasticsearch) [🛠️](roles/svc-db-elasticsearch/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Tor Onion Service](roles/svc-net-tor/) | -| [ERPNext](roles/web-app-erpnext/) | beta | Self-hosted open-source ERP suite for finance, CRM, inventory, manufacturing, HR, sales, and projects | [🌐](https://erpnext.com/) [🎬](https://www.youtube.com/watch?v=JlqE-BfDEUY) [🛠️](roles/web-app-erpnext/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [EspoCRM](roles/web-app-espocrm/) | beta | Open-source CRM platform for managing sales, customer interactions, and business processes across teams | [🌐](https://www.espocrm.com/) [🎬](https://www.youtube.com/watch?v=TH6yJPTPLO0) [🛠️](roles/web-app-espocrm/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [ERPNext](roles/web-app-erpnext/) | beta | Self-hosted open-source ERP suite for finance, CRM, inventory, manufacturing, HR, sales, and projects | [🌐](https://erpnext.com/) [🎬](https://www.youtube.com/watch?v=JlqE-BfDEUY) [🛠️](roles/web-app-erpnext/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [EspoCRM](roles/web-app-espocrm/) | beta | Open-source CRM platform for managing sales, customer interactions, and business processes across teams | [🌐](https://www.espocrm.com/) [🎬](https://www.youtube.com/watch?v=TH6yJPTPLO0) [🛠️](roles/web-app-espocrm/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Fediwall](roles/web-app-fediwall/) | beta | Self-hosted Fediverse wall for showing recent public posts by hashtag or account in a live masonry grid | [🌐](https://github.com/defnull/fediwall) [🛠️](roles/web-app-fediwall/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Friendica](roles/web-app-friendica/), [Mastodon](roles/web-app-mastodon/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | -| [Fider](roles/web-app-fider/) | beta | Open-source community feedback platform for collecting, voting on, and prioritizing feature requests and ideas | [🌐](https://www.fider.io/) [🛠️](roles/web-app-fider/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Fider](roles/web-app-fider/) | beta | Open-source community feedback platform for collecting, voting on, and prioritizing feature requests and ideas | [🌐](https://www.fider.io/) [🛠️](roles/web-app-fider/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [File Server](roles/web-svc-file/) | beta | Static file server over HTTPS with directory listing, .well-known handling, and automatic TLS certificates | [🛠️](roles/web-svc-file/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [Firewall ruleset](roles/svc-net-firewall/) | beta | nftables packet filter whose rules each role owns as its own table, replaced atomically and reloaded at boot | [🌐](https://www.netfilter.org/projects/nftables/) [🎬](https://www.youtube.com/watch?v=EGKhIljDPCw) [🛠️](roles/svc-net-firewall/README.md#quick-setup) | | -| [Flowise](roles/web-app-flowise/) | beta | Low-code visual builder for creating, testing, and publishing LLM and RAG workflows as HTTP endpoints | [🌐](https://flowiseai.com/) [🎬](https://www.youtube.com/watch?v=h_0TJTY6Ulg) [🛠️](roles/web-app-flowise/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Ollama](roles/svc-ai-ollama/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Qdrant](roles/svc-db-qdrant/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Friendica](roles/web-app-friendica/) | beta | Decentralized, federated social-networking platform for peer-to-peer communication and community building | [🌐](https://friendi.ca/) [🛠️](roles/web-app-friendica/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Funkwhale](roles/web-app-funkwhale/) | beta | Self-hosted, federated platform for sharing, hosting, and streaming your own music and audio library | [🌐](https://www.funkwhale.audio/) [🛠️](roles/web-app-funkwhale/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Typesense](roles/svc-db-typesense/), [Universal Logout](roles/web-svc-logout/) | -| [FusionDirectory](roles/web-app-fusiondirectory/) | beta | Web-based LDAP administration interface for managing users, groups, and directory objects | [🌐](https://www.fusiondirectory.org/) [🎬](https://www.youtube.com/watch?v=dThpssnXGyk) [🛠️](roles/web-app-fusiondirectory/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Gitea](roles/web-app-gitea/) | beta | Lightweight self-hosted Git service for code collaboration, version control, and repository management | [🌐](https://about.gitea.com/) [🛠️](roles/web-app-gitea/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [GitLab](roles/web-app-gitlab/) | beta | Self-hosted DevOps platform for Git source-code management, CI/CD pipelines, and issue tracking | [🌐](https://about.gitlab.com/) [🎬](https://www.youtube.com/watch?v=7q9Y1Cv-ib0) [🛠️](roles/web-app-gitlab/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Flowise](roles/web-app-flowise/) | beta | Low-code visual builder for creating, testing, and publishing LLM and RAG workflows as HTTP endpoints | [🌐](https://flowiseai.com/) [🎬](https://www.youtube.com/watch?v=h_0TJTY6Ulg) [🛠️](roles/web-app-flowise/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Matomo](roles/web-app-matomo/), [Ollama](roles/svc-ai-ollama/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Qdrant](roles/svc-db-qdrant/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Friendica](roles/web-app-friendica/) | beta | Decentralized, federated social-networking platform for peer-to-peer communication and community building | [🌐](https://friendi.ca/) [🛠️](roles/web-app-friendica/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Funkwhale](roles/web-app-funkwhale/) | beta | Self-hosted, federated platform for sharing, hosting, and streaming your own music and audio library | [🌐](https://www.funkwhale.audio/) [🛠️](roles/web-app-funkwhale/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Typesense](roles/svc-db-typesense/), [Universal Logout](roles/web-svc-logout/) | +| [FusionDirectory](roles/web-app-fusiondirectory/) | beta | Web-based LDAP administration interface for managing users, groups, and directory objects | [🌐](https://www.fusiondirectory.org/) [🎬](https://www.youtube.com/watch?v=dThpssnXGyk) [🛠️](roles/web-app-fusiondirectory/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Gitea](roles/web-app-gitea/) | beta | Lightweight self-hosted Git service for code collaboration, version control, and repository management | [🌐](https://about.gitea.com/) [🛠️](roles/web-app-gitea/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [GitLab](roles/web-app-gitlab/) | beta | Self-hosted DevOps platform for Git source-code management, CI/CD pipelines, and issue tracking | [🌐](https://about.gitlab.com/) [🎬](https://www.youtube.com/watch?v=7q9Y1Cv-ib0) [🛠️](roles/web-app-gitlab/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Jenkins](roles/web-app-jenkins/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Matrix](roles/web-app-matrix/), [Mattermost](roles/web-app-mattermost/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Hermes Agent](roles/web-app-hermes/) | beta | Kernel-isolated agent runtime serving an OpenAI-compatible API, routed to models through the shared LLM gateway | [🌐](https://hermes-agent.nousresearch.com/) [🛠️](roles/web-app-hermes/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Kata & gVisor](roles/svc-virt-kata/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [Home Assistant](roles/web-app-homeassistant/) | beta | Smart-home automation hub with device integrations, dashboards and a built-in MCP server for AI agent access | [🌐](https://www.home-assistant.io/) [🛠️](roles/web-app-homeassistant/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Dashboard](roles/web-app-dashboard/), [Flowise](roles/web-app-flowise/), [Hermes Agent](roles/web-app-hermes/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [HTML Server](roles/web-svc-html/) | beta | NGINX vhost for serving static HTML websites over HTTPS with automatic Let's Encrypt certificates | [🛠️](roles/web-svc-html/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | @@ -114,63 +115,63 @@ Every invokable role in the tested lifecycle envelope, with its upstream homepag | [Imprint](roles/web-svc-legal/) | beta | NGINX-served static page rendering a legal imprint (Impressum) from a Jinja2 template | [🛠️](roles/web-svc-legal/README.md#quick-setup) | [HTML Server](roles/web-svc-html/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [Intel Driver](roles/drv-intel/) | alpha | Intel VA-API media driver for hardware-accelerated video decode and encode across Linux distributions | [🌐](https://github.com/intel/media-driver) [🛠️](roles/drv-intel/README.md#quick-setup) | | | [Jellyfin](roles/web-app-jellyfin/) | beta | Self-hosted media server for streaming and organizing movies, TV, music, and photos, with OIDC and LDAP login | [🌐](https://jellyfin.org/) [🎬](https://www.youtube.com/watch?v=0t3lSJL3B_Q) [🛠️](roles/web-app-jellyfin/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Jenkins](roles/web-app-jenkins/) | beta | Open-source automation server for CI/CD pipelines, builds, and plugin-driven deployment workflows | [🌐](https://www.jenkins.io/) [🎬](https://www.youtube.com/watch?v=rNy00_gOo8o) [🛠️](roles/web-app-jenkins/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Jitsi Meet](roles/web-app-jitsi/) | beta | Self-hosted WebRTC platform for end-to-end-encryptable video conferencing with federated authentication | [🌐](https://jitsi.org/) [🎬](https://www.youtube.com/watch?v=TB7LlM4erx8) [🛠️](roles/web-app-jitsi/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Universal Logout](roles/web-svc-logout/) | -| [Joomla](roles/web-app-joomla/) | beta | Open-source content management system for building websites, intranets, and online applications without writing code | [🌐](https://www.joomla.org/) [🎬](https://www.youtube.com/watch?v=mXFSIC_zM2o) [🛠️](roles/web-app-joomla/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Jenkins](roles/web-app-jenkins/) | beta | Open-source automation server for CI/CD pipelines, builds, and plugin-driven deployment workflows | [🌐](https://www.jenkins.io/) [🎬](https://www.youtube.com/watch?v=rNy00_gOo8o) [🛠️](roles/web-app-jenkins/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Gitea](roles/web-app-gitea/), [GitLab](roles/web-app-gitlab/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Mattermost](roles/web-app-mattermost/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Jitsi Meet](roles/web-app-jitsi/) | beta | Self-hosted WebRTC platform for end-to-end-encryptable video conferencing with federated authentication | [🌐](https://jitsi.org/) [🎬](https://www.youtube.com/watch?v=TB7LlM4erx8) [🛠️](roles/web-app-jitsi/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Universal Logout](roles/web-svc-logout/) | +| [Joomla](roles/web-app-joomla/) | beta | Open-source content management system for building websites, intranets, and online applications without writing code | [🌐](https://www.joomla.org/) [🎬](https://www.youtube.com/watch?v=mXFSIC_zM2o) [🛠️](roles/web-app-joomla/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Jupyter](roles/svc-ai-jupyter/) | alpha | Kernel backend that runs the code a chat writes, in a container instead of in the browser sandbox | [🌐](https://jupyter.org/) [🛠️](roles/svc-ai-jupyter/README.md#quick-setup) | | | [Kata & gVisor](roles/svc-virt-kata/) | beta | Kernel-isolating container runtimes that sandbox untrusted workloads in a microVM or a userspace kernel | [🌐](https://katacontainers.io/) [🛠️](roles/svc-virt-kata/README.md#quick-setup) | | -| [Keycloak](roles/web-app-keycloak/) | beta | Self-hosted identity and access management platform for single sign-on, multi-factor authentication, and user federation | [🌐](https://www.keycloak.org/) [🛠️](roles/web-app-keycloak/README.md#quick-setup) | [Content Delivery Network](roles/web-svc-cdn/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | -| [KIX](roles/web-app-kix/) | beta | Open-source ITSM and helpdesk platform (OTRS fork) for ticketing, CMDB, knowledge base, and reporting | [🌐](https://www.kixdesk.com/en/) [🎬](https://www.youtube.com/watch?v=D62emos1FQ8) [🛠️](roles/web-app-kix/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Keycloak](roles/web-app-keycloak/) | beta | Self-hosted identity and access management platform for single sign-on, multi-factor authentication, and user federation | [🌐](https://www.keycloak.org/) [🛠️](roles/web-app-keycloak/README.md#quick-setup) | [Content Delivery Network](roles/web-svc-cdn/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/) | +| [KIX](roles/web-app-kix/) | beta | Open-source ITSM and helpdesk platform (OTRS fork) for ticketing, CMDB, knowledge base, and reporting | [🌐](https://www.kixdesk.com/en/) [🎬](https://www.youtube.com/watch?v=D62emos1FQ8) [🛠️](roles/web-app-kix/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [LAM](roles/web-app-lam/) | beta | Web interface for administering LDAP directories: managing users, groups, and other directory objects | [🌐](https://www.ldap-account-manager.org/) [🛠️](roles/web-app-lam/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [LibreTranslate](roles/web-svc-libretranslate/) | beta | Self-hosted open-source machine translation API for translating and detecting text across languages | [🌐](https://libretranslate.com/) [🛠️](roles/web-svc-libretranslate/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [LibreTranslate Engine](roles/svc-ai-libretranslate-engine/) | beta | Machine translation engine without a web surface, for pipelines that call it directly | [🌐](https://libretranslate.com/) [🛠️](roles/svc-ai-libretranslate-engine/README.md#quick-setup) | | | [LID Switch Driver](roles/drv-lid-switch/) | alpha | Power-aware systemd logind configuration for predictable laptop lid-close behavior across battery, AC, and docked states | [🛠️](roles/drv-lid-switch/README.md#quick-setup) | | -| [Listmonk](roles/web-app-listmonk/) | beta | Self-hosted platform for newsletters and mailing lists with subscriber segmentation and campaign analytics | [🌐](https://listmonk.app/) [🛠️](roles/web-app-listmonk/README.md#quick-setup) | [Assets Server](roles/web-svc-asset/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Listmonk](roles/web-app-listmonk/) | beta | Self-hosted platform for newsletters and mailing lists with subscriber segmentation and campaign analytics | [🌐](https://listmonk.app/) [🛠️](roles/web-app-listmonk/README.md#quick-setup) | [Assets Server](roles/web-svc-asset/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [LiteLLM Admin UI](roles/web-app-litellm/) | beta | Browser-facing admin interface for the shared LLM gateway, published on its own domain behind the front proxy | [🌐](https://docs.litellm.ai/) [🛠️](roles/web-app-litellm/README.md#quick-setup) | [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [LiteLLM Gateway](roles/svc-ai-litellm/) | beta | OpenAI-compatible LLM gateway routing a single /v1 endpoint to local and hosted model backends with per-consumer keys | [🌐](https://docs.litellm.ai/) [🛠️](roles/svc-ai-litellm/README.md#quick-setup) | [LM Studio](roles/svc-ai-lmstudio/), [Ollama](roles/svc-ai-ollama/), [PostgreSQL](roles/svc-db-postgres/), [System One](roles/svc-ai-s1/) | | [LittleJS](roles/web-app-littlejs/) | alpha | Self-hosted HTML5 game sandbox for browsing and running lightweight engine demos and games in the browser | [🌐](https://killedbyapixel.github.io/LittleJS/) [🎬](https://youtu.be/chuBzGjv7Ms) [🛠️](roles/web-app-littlejs/README.md#quick-setup) | [Content Delivery Network](roles/web-svc-cdn/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [LM Studio](roles/svc-ai-lmstudio/) | beta | Headless local model server for open LLMs with an OpenAI-compatible API on the container network | [🌐](https://lmstudio.ai/) [🛠️](roles/svc-ai-lmstudio/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/) | -| [Magento](roles/web-app-magento/) | alpha | Self-hosted e-commerce platform for catalog management, checkout, promotions, and multi-store online retail | [🌐](https://business.adobe.com/products/commerce/magento/open-source.html) [🎬](https://www.youtube.com/watch?v=nFdz9DZrdFc) [🛠️](roles/web-app-magento/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Mailu](roles/web-app-mailu/) | beta | Self-hosted mail server suite with SMTP/IMAP/POP3, webmail, spam filtering, antivirus, DKIM signing, and TLS | [🌐](https://mailu.io/) [🛠️](roles/web-app-mailu/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [LTEngine](roles/svc-ai-ltengine/) | alpha | LLM-backed translation engine speaking the LibreTranslate API, for pipelines that call it directly | [🌐](https://github.com/LibreTranslate/LTEngine) [🛠️](roles/svc-ai-ltengine/README.md#quick-setup) | | +| [Magento](roles/web-app-magento/) | alpha | Self-hosted e-commerce platform for catalog management, checkout, promotions, and multi-store online retail | [🌐](https://business.adobe.com/products/commerce/magento/open-source.html) [🎬](https://www.youtube.com/watch?v=nFdz9DZrdFc) [🛠️](roles/web-app-magento/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [MariaDB](roles/svc-db-mariadb/) | beta | Shared relational database server for applications needing central SQL storage, run in Docker | [🌐](https://mariadb.org/) [🎬](https://www.youtube.com/watch?v=YsIJB15fq_c) [🛠️](roles/svc-db-mariadb/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Tor Onion Service](roles/svc-net-tor/) | -| [Mastodon](roles/web-app-mastodon/) | beta | Self-hosted federated social network for microblogging, real-time timelines, and community moderation | [🌐](https://joinmastodon.org/) [🛠️](roles/web-app-mastodon/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Mastodon](roles/web-app-mastodon/) | beta | Self-hosted federated social network for microblogging, real-time timelines, and community moderation | [🌐](https://joinmastodon.org/) [🛠️](roles/web-app-mastodon/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Matomo](roles/web-app-matomo/) | beta | Self-hosted, privacy-first web analytics platform for real-time visitor tracking and site metrics | [🌐](https://matomo.org/) [🎬](https://www.youtube.com/watch?v=Qc2kooLNDiU) [🛠️](roles/web-app-matomo/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [MariaDB](roles/svc-db-mariadb/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Matrix](roles/web-app-matrix/) | beta | Federated, end-to-end-encrypted messaging platform with a web client and protocol bridges to external chat networks | [🌐](https://matrix.org/) [🎬](https://www.youtube.com/watch?v=3AVsNqH_-9M) [🛠️](roles/web-app-matrix/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Coturn](roles/web-svc-coturn/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Mattermost](roles/web-app-mattermost/) | beta | Open-source, self-hosted team messaging platform for real-time collaboration via channels, threads, and integrations | [🌐](https://mattermost.com/) [🎬](https://www.youtube.com/watch?v=szfZfQFUSnU) [🛠️](roles/web-app-mattermost/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Content Delivery Network](roles/web-svc-cdn/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Matrix](roles/web-app-matrix/) | beta | Federated, end-to-end-encrypted messaging platform with a web client and protocol bridges to external chat networks | [🌐](https://matrix.org/) [🎬](https://www.youtube.com/watch?v=3AVsNqH_-9M) [🛠️](roles/web-app-matrix/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Coturn](roles/web-svc-coturn/), [Dashboard](roles/web-app-dashboard/), [GitLab](roles/web-app-gitlab/), [Jitsi Meet](roles/web-app-jitsi/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Matomo](roles/web-app-matomo/), [n8n](roles/web-app-n8n/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Mattermost](roles/web-app-mattermost/) | beta | Open-source, self-hosted team messaging platform for real-time collaboration via channels, threads, and integrations | [🌐](https://mattermost.com/) [🎬](https://www.youtube.com/watch?v=szfZfQFUSnU) [🛠️](roles/web-app-mattermost/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Content Delivery Network](roles/web-svc-cdn/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [GitLab](roles/web-app-gitlab/), [Hermes Agent](roles/web-app-hermes/), [Jenkins](roles/web-app-jenkins/), [Jitsi Meet](roles/web-app-jitsi/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [MCP Adapter](roles/svc-ai-mcp-adapter/) | alpha | Reusable per-provider MCP adapter exposing a narrow allowlisted read surface | [🌐](https://modelcontextprotocol.io/) [🛠️](roles/svc-ai-mcp-adapter/README.md#quick-setup) | | -| [MediaWiki](roles/web-app-mediawiki/) | beta | Collaborative wiki platform for user-driven documentation with an extensible extension ecosystem and content management | [🌐](https://www.mediawiki.org/) [🎬](https://www.youtube.com/watch?v=-JnIjpRvgNY) [🛠️](roles/web-app-mediawiki/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [MediaWiki](roles/web-app-mediawiki/) | beta | Collaborative wiki platform for user-driven documentation with an extensible extension ecosystem and content management | [🌐](https://www.mediawiki.org/) [🎬](https://www.youtube.com/watch?v=-JnIjpRvgNY) [🛠️](roles/web-app-mediawiki/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Discourse](roles/web-app-discourse/), [Keycloak](roles/web-app-keycloak/), [LibreTranslate](roles/web-svc-libretranslate/), [LiteLLM Gateway](roles/svc-ai-litellm/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [PeerTube](roles/web-app-peertube/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Memcached](roles/svc-db-memcached/) | beta | Central in-memory caching engine with per-consumer key-prefix isolation, usable centrally or as an embedded sidecar | [🌐](https://memcached.org/) [🛠️](roles/svc-db-memcached/README.md#quick-setup) | [Tor Onion Service](roles/svc-net-tor/) | | [Meta](roles/web-app-meta/) | alpha | Interactive graph visualization of all Infinito.Nexus roles and their dependencies for architecture transparency | [🛠️](roles/web-app-meta/README.md#quick-setup) | [API](roles/web-svc-api/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [Mini-QR](roles/web-app-mini-qr/) | beta | Self-hosted browser tool for generating QR codes from text or URLs with privacy-friendly local rendering | [🌐](https://github.com/lyqht/mini-qr) [🛠️](roles/web-app-mini-qr/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | -| [Mobilizon](roles/web-app-mobilizon/) | beta | Federated, self-hosted platform for organizing events and building communities and groups | [🌐](https://joinmobilizon.org/) [🎬](https://framatube.org/w/2mtugeJXaihsQnTHVWHf2E) [🛠️](roles/web-app-mobilizon/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Moodle](roles/web-app-moodle/) | beta | Open-source learning management system for creating, managing, and delivering online courses and e-learning | [🌐](https://moodle.org/) [🎬](https://www.youtube.com/watch?v=3ORsUGVNxGs) [🛠️](roles/web-app-moodle/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [n8n](roles/web-app-n8n/) | beta | Self-hosted low-code automation platform for connecting services and building workflow integrations visually | [🌐](https://n8n.io/) [🎬](https://youtu.be/4cQWJViybAQ) [🛠️](roles/web-app-n8n/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Nextcloud](roles/web-app-nextcloud/) | beta | Self-hosted cloud platform for file sync, sharing, real-time collaboration, and team workspaces under your own domain | [🌐](https://nextcloud.com/) [🎬](https://youtu.be/3jcYJGQgenI?si=FDmoMSrAb9_WvviC) [🛠️](roles/web-app-nextcloud/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [BigBlueButton](roles/web-app-bigbluebutton/), [Collabora](roles/web-svc-collabora/), [Corporate Design](roles/web-svc-css/), [Coturn](roles/web-svc-coturn/), [Dashboard](roles/web-app-dashboard/), [Discourse](roles/web-app-discourse/), [GitLab](roles/web-app-gitlab/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Mastodon](roles/web-app-mastodon/), [Matomo](roles/web-app-matomo/), [Matrix](roles/web-app-matrix/), [Mattermost](roles/web-app-mattermost/), [OnlyOffice](roles/web-svc-onlyoffice/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [OpenProject](roles/web-app-openproject/), [PeerTube](roles/web-app-peertube/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/), [XWiki](roles/web-app-xwiki/), [Zammad](roles/web-app-zammad/) | +| [Mobilizon](roles/web-app-mobilizon/) | beta | Federated, self-hosted platform for organizing events and building communities and groups | [🌐](https://joinmobilizon.org/) [🎬](https://framatube.org/w/2mtugeJXaihsQnTHVWHf2E) [🛠️](roles/web-app-mobilizon/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Moodle](roles/web-app-moodle/) | beta | Open-source learning management system for creating, managing, and delivering online courses and e-learning | [🌐](https://moodle.org/) [🎬](https://www.youtube.com/watch?v=3ORsUGVNxGs) [🛠️](roles/web-app-moodle/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [BigBlueButton](roles/web-app-bigbluebutton/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Jitsi Meet](roles/web-app-jitsi/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Matrix](roles/web-app-matrix/), [Nextcloud](roles/web-app-nextcloud/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [n8n](roles/web-app-n8n/) | beta | Self-hosted low-code automation platform for connecting services and building workflow integrations visually | [🌐](https://n8n.io/) [🎬](https://youtu.be/4cQWJViybAQ) [🛠️](roles/web-app-n8n/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Nextcloud](roles/web-app-nextcloud/) | beta | Self-hosted cloud platform for file sync, sharing, real-time collaboration, and team workspaces under your own domain | [🌐](https://nextcloud.com/) [🎬](https://youtu.be/3jcYJGQgenI?si=FDmoMSrAb9_WvviC) [🛠️](roles/web-app-nextcloud/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [BigBlueButton](roles/web-app-bigbluebutton/), [Collabora](roles/web-svc-collabora/), [Corporate Design](roles/web-svc-css/), [Coturn](roles/web-svc-coturn/), [Dashboard](roles/web-app-dashboard/), [Discourse](roles/web-app-discourse/), [GitLab](roles/web-app-gitlab/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [MariaDB](roles/svc-db-mariadb/), [Mastodon](roles/web-app-mastodon/), [Matomo](roles/web-app-matomo/), [Matrix](roles/web-app-matrix/), [Mattermost](roles/web-app-mattermost/), [OnlyOffice](roles/web-svc-onlyoffice/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [OpenProject](roles/web-app-openproject/), [PeerTube](roles/web-app-peertube/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/), [XWiki](roles/web-app-xwiki/), [Zammad](roles/web-app-zammad/) | | [NFS Client](roles/svc-storage-nfs-client/) | beta | Client-side NFS setup for mounting a remote server export on every swarm node, with a deploy-time reachability probe | [🛠️](roles/svc-storage-nfs-client/README.md#quick-setup) | [NFS Server](roles/svc-storage-nfs-server/) | | [NFS Server](roles/svc-storage-nfs-server/) | beta | Kernel-level NFS server for exporting a shared directory tree with IP-restricted access for swarm-node hosts | [🛠️](roles/svc-storage-nfs-server/README.md#quick-setup) | [Backup NFS](roles/svc-bkp-nfs-2-local/) | -| [Odoo ERP](roles/web-app-odoo/) | beta | Open-source ERP suite for CRM, sales, accounting, inventory, HR, and website management | [🌐](https://www.odoo.com/) [🎬](https://www.youtube.com/watch?v=FCmFgHVzd3E) [🛠️](roles/web-app-odoo/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Odoo ERP](roles/web-app-odoo/) | beta | Open-source ERP suite for CRM, sales, accounting, inventory, HR, and website management | [🌐](https://www.odoo.com/) [🎬](https://www.youtube.com/watch?v=FCmFgHVzd3E) [🛠️](roles/web-app-odoo/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Nextcloud](roles/web-app-nextcloud/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Ollama](roles/svc-ai-ollama/) | beta | Self-hosted model server for running open LLMs locally behind a simple HTTP API | [🌐](https://ollama.com/) [🛠️](roles/svc-ai-ollama/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/) | | [OnlyOffice](roles/web-svc-onlyoffice/) | beta | Document server backend for real-time in-browser editing of documents, spreadsheets, and presentations | [🌐](https://www.onlyoffice.com/) [🎬](https://www.youtube.com/watch?v=uiFxnRK_aWM) [🛠️](roles/web-svc-onlyoffice/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Tor Onion Service](roles/svc-net-tor/) | -| [Open WebUI](roles/web-app-openwebui/) | beta | Self-hosted chat interface for interacting with local and private AI models via backends such as Ollama | [🌐](https://openwebui.com/) [🛠️](roles/web-app-openwebui/README.md#quick-setup) | [Agent Broker](roles/svc-ai-agent-broker/), [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Qdrant](roles/svc-db-qdrant/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Open WebUI](roles/web-app-openwebui/) | beta | Self-hosted chat interface for interacting with local and private AI models via backends such as Ollama | [🌐](https://openwebui.com/) [🛠️](roles/web-app-openwebui/README.md#quick-setup) | [Agent Broker](roles/svc-ai-agent-broker/), [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Qdrant](roles/svc-db-qdrant/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [OpenBao](roles/web-app-openbao/) | beta | Identity-based secrets, PKI and encryption manager with OIDC login, group-mapped policies and static-seal auto-unseal | [🌐](https://openbao.org/) [🎬](https://www.youtube.com/watch?v=hPBT-kSS-68) [🛠️](roles/web-app-openbao/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [OpenClaw](roles/web-app-openclaw/) | beta | Self-hosted personal AI agent gateway with a browser control UI, sandboxed in a Kata microVM or gVisor runtime | [🌐](https://openclaw.ai/) [🛠️](roles/web-app-openclaw/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Kata & gVisor](roles/svc-virt-kata/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [OpenCloud](roles/web-app-opencloud/) | beta | Self-hosted platform for file sync, sharing, and collaboration with Keycloak SSO and central LDAP directory | [🌐](https://opencloud.eu/) [🎬](https://www.youtube.com/watch?v=Cg48rFNY0R4) [🛠️](roles/web-app-opencloud/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [OpenCloud](roles/web-app-opencloud/) | beta | Self-hosted platform for file sync, sharing, and collaboration with Keycloak SSO and central LDAP directory | [🌐](https://opencloud.eu/) [🎬](https://www.youtube.com/watch?v=Cg48rFNY0R4) [🛠️](roles/web-app-opencloud/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [OpenLDAP](roles/svc-db-openldap/) | beta | Self-hosted LDAP directory service for centralized user, group, and schema management with TLS-secured access | [🌐](https://www.openldap.org/) [🛠️](roles/svc-db-openldap/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Tor Onion Service](roles/svc-net-tor/) | -| [OpenProject](roles/web-app-openproject/) | beta | Self-hosted platform for project management, work-package tracking, Gantt planning, and team collaboration | [🌐](https://www.openproject.org/) [🎬](https://www.youtube.com/watch?v=rvCd0xH7rCs) [🛠️](roles/web-app-openproject/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [OpenProject](roles/web-app-openproject/) | beta | Self-hosted platform for project management, work-package tracking, Gantt planning, and team collaboration | [🌐](https://www.openproject.org/) [🎬](https://www.youtube.com/watch?v=rvCd0xH7rCs) [🛠️](roles/web-app-openproject/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [GitLab](roles/web-app-gitlab/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Nextcloud](roles/web-app-nextcloud/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [OpenResty](roles/svc-prx-openresty/) | beta | NGINX-based web platform and reverse proxy with validated config reloads and Let's Encrypt integration | [🌐](https://openresty.org/) [🎬](https://www.youtube.com/watch?v=NR8U1qYHGqk) [🛠️](roles/svc-prx-openresty/README.md#quick-setup) | [Tor Onion Service](roles/svc-net-tor/) | -| [OpenTalk](roles/web-app-opentalk/) | beta | Sovereign open-source video conferencing platform for audio/video calls, screen sharing, breakout rooms, and SSO | [🌐](https://opentalk.eu/) [🛠️](roles/web-app-opentalk/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Coturn](roles/web-svc-coturn/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Universal Logout](roles/web-svc-logout/) | +| [OpenTalk](roles/web-app-opentalk/) | beta | Sovereign open-source video conferencing platform for audio/video calls, screen sharing, breakout rooms, and SSO | [🌐](https://opentalk.eu/) [🛠️](roles/web-app-opentalk/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Coturn](roles/web-svc-coturn/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Universal Logout](roles/web-svc-logout/) | | [Package cache](roles/svc-cache-package/) | beta | Cluster-local proxy cache for package upstreams, serving apt, pip, npm and raw artefacts every role declares | [🛠️](roles/svc-cache-package/README.md#quick-setup) | [Docker Swarm](roles/svc-swarm-node/) | | [Package update](roles/update/) | beta | System-maintenance role for refreshing the package index and upgrading installed packages via the OS package manager | [🛠️](roles/update/README.md#quick-setup) | | -| [PeerTube](roles/web-app-peertube/) | beta | Decentralized, federated platform for peer-to-peer video hosting, streaming, and live broadcasting | [🌐](https://joinpeertube.org/) [🎬](https://framatube.org/w/cX4KAPLTwha6hzLZ5xePGZ) [🛠️](roles/web-app-peertube/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Universal Logout](roles/web-svc-logout/) | -| [Penpot](roles/web-app-penpot/) | beta | Self-hosted design and prototyping platform for UI/UX work, real-time collaboration, and developer handoff | [🌐](https://penpot.app/) [🎬](https://www.youtube.com/watch?v=vAkb4QQmbfk) [🛠️](roles/web-app-penpot/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [pgAdmin](roles/web-app-pgadmin/) | beta | Web-based administration and development platform for managing multiple PostgreSQL servers, with optional OAuth2 login | [🌐](https://www.pgadmin.org/) [🎬](https://www.youtube.com/watch?v=WFT5MaZN6g4) [🛠️](roles/web-app-pgadmin/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [PeerTube](roles/web-app-peertube/) | beta | Decentralized, federated platform for peer-to-peer video hosting, streaming, and live broadcasting | [🌐](https://joinpeertube.org/) [🎬](https://framatube.org/w/cX4KAPLTwha6hzLZ5xePGZ) [🛠️](roles/web-app-peertube/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Universal Logout](roles/web-svc-logout/) | +| [Penpot](roles/web-app-penpot/) | beta | Self-hosted design and prototyping platform for UI/UX work, real-time collaboration, and developer handoff | [🌐](https://penpot.app/) [🎬](https://www.youtube.com/watch?v=vAkb4QQmbfk) [🛠️](roles/web-app-penpot/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [pgAdmin](roles/web-app-pgadmin/) | beta | Web-based administration and development platform for managing multiple PostgreSQL servers, with optional OAuth2 login | [🌐](https://www.pgadmin.org/) [🎬](https://www.youtube.com/watch?v=WFT5MaZN6g4) [🛠️](roles/web-app-pgadmin/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [PhpMyAdmin](roles/web-app-phpmyadmin/) | beta | Web-based administration tool for MySQL and MariaDB with schema management, querying, and import/export | [🌐](https://www.phpmyadmin.net/) [🛠️](roles/web-app-phpmyadmin/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Pi Coding Agent](roles/dsk-gnt-pi/) | alpha | Minimal terminal coding agent with a lazy-loading skill system, wired to the platform's own model gateway | [🌐](https://github.com/earendil-works/pi) [🛠️](roles/dsk-gnt-pi/README.md#quick-setup) | [Base](roles/dsk-base/), [LiteLLM Gateway](roles/svc-ai-litellm/) | -| [Pixelfed](roles/web-app-pixelfed/) | beta | Decentralized, federated platform for photo sharing and image feeds across the fediverse | [🌐](https://pixelfed.org/) [🛠️](roles/web-app-pixelfed/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Pixelfed](roles/web-app-pixelfed/) | beta | Decentralized, federated platform for photo sharing and image feeds across the fediverse | [🌐](https://pixelfed.org/) [🛠️](roles/web-app-pixelfed/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [PostgreSQL](roles/svc-db-postgres/) | beta | Containerized relational database with automated database, user, and privilege provisioning | [🌐](https://www.postgresql.org/) [🛠️](roles/svc-db-postgres/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Tor Onion Service](roles/svc-net-tor/) | | [Postmarks](roles/web-app-postmarks/) | beta | Single-user ActivityPub bookmarking site federating public bookmarks to Mastodon and the Fediverse | [🌐](https://github.com/ckolderup/postmarks) [🛠️](roles/web-app-postmarks/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Pretix](roles/web-app-pretix/) | beta | Self-hosted ticketing platform for events with online sales, registration management, and secure payments | [🌐](https://pretix.eu/) [🎬](https://www.youtube.com/watch?v=5F6QkZP1Zf0) [🛠️](roles/web-app-pretix/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Prometheus](roles/web-app-prometheus/) | beta | Open-source monitoring and alerting toolkit for storing metrics as time-series data and querying them with PromQL | [🌐](https://prometheus.io/) [🎬](https://www.youtube.com/watch?v=PzFUwBflXYc) [🛠️](roles/web-app-prometheus/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Flowise](roles/web-app-flowise/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Pretix](roles/web-app-pretix/) | beta | Self-hosted ticketing platform for events with online sales, registration management, and secure payments | [🌐](https://pretix.eu/) [🎬](https://www.youtube.com/watch?v=5F6QkZP1Zf0) [🛠️](roles/web-app-pretix/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Prometheus](roles/web-app-prometheus/) | beta | Open-source monitoring and alerting toolkit for storing metrics as time-series data and querying them with PromQL | [🌐](https://prometheus.io/) [🎬](https://www.youtube.com/watch?v=PzFUwBflXYc) [🛠️](roles/web-app-prometheus/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Flowise](roles/web-app-flowise/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Proprietary Drivers](roles/drv-non-free/) | alpha | Proprietary GPU driver installer for Arch-based systems via mhwd nonfree hardware detection | [🛠️](roles/drv-non-free/README.md#quick-setup) | | | [Qdrant](roles/svc-db-qdrant/) | beta | Central vector database engine for storing and searching per-consumer embeddings shared across the stack | [🌐](https://qdrant.tech/) [🎬](https://www.youtube.com/watch?v=_opNdOa48Lc) [🛠️](roles/svc-db-qdrant/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Flowise](roles/web-app-flowise/), [Hermes Agent](roles/web-app-hermes/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [Tor Onion Service](roles/svc-net-tor/) | | [RabbitMQ](roles/svc-db-rabbitmq/) | beta | Shared message broker for async messaging with per-consumer virtual host and user isolation | [🌐](https://www.rabbitmq.com/) [🛠️](roles/svc-db-rabbitmq/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Tor Onion Service](roles/svc-net-tor/) | @@ -178,34 +179,39 @@ Every invokable role in the tested lifecycle envelope, with its upstream homepag | [Redis](roles/svc-db-redis/) | beta | In-memory key-value cache and store, shared centrally with per-consumer ACL isolation or embedded as a sidecar | [🌐](https://redis.io/) [🛠️](roles/svc-db-redis/README.md#quick-setup) | [Tor Onion Service](roles/svc-net-tor/) | | [Roulette Wheel](roles/web-app-roulette-wheel/) | alpha | Browser-based spinning wheel app for decision making, random selection, and gamification | [🌐](https://github.com/p-wojt/roulette-wheel) [🛠️](roles/web-app-roulette-wheel/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [Searxng](roles/svc-ai-searxng/) | alpha | Metasearch engine the platform queries for web results without handing a user's query to a tracking search provider | [🌐](https://docs.searxng.org/) [🛠️](roles/svc-ai-searxng/README.md#quick-setup) | | -| [SeaweedFS](roles/web-svc-seaweedfs/) | beta | Distributed S3-compatible object store for shared, per-consumer bucket provisioning across web-app roles | [🌐](https://seaweedfs.org/) [🛠️](roles/web-svc-seaweedfs/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Prometheus](roles/web-app-prometheus/), [Universal Logout](roles/web-svc-logout/) | +| [SeaweedFS](roles/web-svc-seaweedfs/) | beta | Distributed S3-compatible object store for shared, per-consumer bucket provisioning across web-app roles | [🌐](https://seaweedfs.org/) [🎬](https://www.youtube.com/watch?v=n0hRleSLAcc) [🛠️](roles/web-svc-seaweedfs/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Prometheus](roles/web-app-prometheus/) | +| [SeaweedFS Console](roles/web-app-seaweedfs/) | beta | Browser administration interface for the SeaweedFS object store, serving the filer and master consoles behind SSO | [🌐](https://seaweedfs.org/) [🎬](https://www.youtube.com/watch?v=n0hRleSLAcc) [🛠️](roles/web-app-seaweedfs/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Self-Hosted Runner](roles/svc-runner/) | beta | Self-hosted CI runner infrastructure for executing GitHub Actions workflow jobs in ephemeral Docker containers | [🌐](https://github.com/actions/runner) [🛠️](roles/svc-runner/README.md#quick-setup) | | -| [Semaphore UI](roles/web-app-semaphore/) | beta | Web UI and API for running Ansible, Terraform/OpenTofu, and shell automation on a schedule or on demand | [🌐](https://semaphoreui.com/) [🎬](https://www.youtube.com/watch?v=nzR4yDuZQMU) [🛠️](roles/web-app-semaphore/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Shopware](roles/web-app-shopware/) | beta | Open-source ecommerce platform built on Symfony and Vue.js for building and scaling online storefronts | [🌐](https://www.shopware.com/en/) [🎬](https://www.youtube.com/watch?v=bG57TZPYsyw) [🛠️](roles/web-app-shopware/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Semaphore UI](roles/web-app-semaphore/) | beta | Web UI and API for running Ansible, Terraform/OpenTofu, and shell automation on a schedule or on demand | [🌐](https://semaphoreui.com/) [🎬](https://www.youtube.com/watch?v=nzR4yDuZQMU) [🛠️](roles/web-app-semaphore/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Shopware](roles/web-app-shopware/) | beta | Open-source ecommerce platform built on Symfony and Vue.js for building and scaling online storefronts | [🌐](https://www.shopware.com/en/) [🎬](https://www.youtube.com/watch?v=bG57TZPYsyw) [🛠️](roles/web-app-shopware/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Simple Icons](roles/web-svc-simpleicons/) | beta | Containerized icon server for hosting the Simple Icons brand SVG and PNG collection for web and documentation use | [🌐](https://simpleicons.org/) [🛠️](roles/web-svc-simpleicons/README.md#quick-setup) | [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | -| [Snipe‑IT](roles/web-app-snipe-it/) | beta | Open-source IT asset management system for tracking hardware and software inventory | [🌐](https://snipeitapp.com/) [🎬](https://www.youtube.com/watch?v=q0V6FAPc1ls) [🛠️](roles/web-app-snipe-it/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Snipe‑IT](roles/web-app-snipe-it/) | beta | Open-source IT asset management system for tracking hardware and software inventory | [🌐](https://snipeitapp.com/) [🎬](https://www.youtube.com/watch?v=q0V6FAPc1ls) [🛠️](roles/web-app-snipe-it/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Social Home](roles/web-app-socialhome/) | alpha | Privacy-first social platform for households, federating peer-to-peer over WebRTC on an embedded SQLite store | [🌐](https://github.com/social-home-io/socialhome) [🛠️](roles/web-app-socialhome/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Coturn](roles/web-svc-coturn/), [Dashboard](roles/web-app-dashboard/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Stalwart Mail Server](roles/web-app-stalwart/) | beta | All-in-one Stalwart mail server with SMTP, IMAP, JMAP, POP3, ManageSieve, CalDAV, CardDAV and built-in spam filtering | [🌐](https://stalw.art/) [🛠️](roles/web-app-stalwart/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Tor SMTP Gateway (svc-net-tor-smtp)](roles/svc-net-tor-smtp/), [Universal Logout](roles/web-svc-logout/) | | [Storage Optimizer](roles/svc-opt-ssd-hdd/) | beta | Tiering role placing Docker volumes on SSD or HDD by workload type, with symlinks keeping paths consistent | [🛠️](roles/svc-opt-ssd-hdd/README.md#quick-setup) | | -| [SuiteCRM](roles/web-app-suitecrm/) | beta | Open-source CRM platform for sales, service, cases, and campaign management with workflow automation | [🌐](https://suitecrm.com/) [🎬](https://www.youtube.com/watch?v=pOAs_C3Fp_E) [🛠️](roles/web-app-suitecrm/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [SuiteCRM](roles/web-app-suitecrm/) | beta | Open-source CRM platform for sales, service, cases, and campaign management with workflow automation | [🌐](https://suitecrm.com/) [🎬](https://www.youtube.com/watch?v=pOAs_C3Fp_E) [🛠️](roles/web-app-suitecrm/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Swapfile](roles/svc-opt-swapfile/) | beta | On-target automation for creating and activating swap space via a cloned swapfile script | [🛠️](roles/svc-opt-swapfile/README.md#quick-setup) | | | [System One](roles/svc-ai-s1/) | beta | Self-hosted System One classifier that answers typed questions so the gateway can route without generating | [🌐](https://huggingface.co/convaiinnovations/laya) [🎬](https://www.youtube.com/watch?v=UoRNo5LoDRE) [🛠️](roles/svc-ai-s1/README.md#quick-setup) | | | [System One API](roles/web-svc-s1/) | alpha | Public vhost that exposes the System One decision API of svc-ai-s1 | [🛠️](roles/web-svc-s1/README.md#quick-setup) | [Prometheus](roles/web-app-prometheus/), [System One](roles/svc-ai-s1/), [Tor Onion Service](roles/svc-net-tor/) | -| [Taiga](roles/web-app-taiga/) | beta | Open-source agile project management platform for Scrum, Kanban, and hybrid team workflows | [🌐](https://taiga.io/) [🎬](https://www.youtube.com/watch?v=TYPX8nQEU5I) [🛠️](roles/web-app-taiga/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Taiga](roles/web-app-taiga/) | beta | Open-source agile project management platform for Scrum, Kanban, and hybrid team workflows | [🌐](https://taiga.io/) [🎬](https://www.youtube.com/watch?v=TYPX8nQEU5I) [🛠️](roles/web-app-taiga/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Tika](roles/svc-ai-tika/) | alpha | Central document extraction engine that turns uploaded files into text for retrieval-augmented chat | [🌐](https://tika.apache.org/) [🛠️](roles/svc-ai-tika/README.md#quick-setup) | | | [Tor Onion Service](roles/svc-net-tor/) | beta | Tor daemon publishing the node as a v3 onion service | [🛠️](roles/svc-net-tor/README.md#quick-setup) | [Asset Mirror (Privacy Proxy)](roles/web-svc-mirror/) | +| [Tor SMTP Gateway (svc-net-tor-smtp)](roles/svc-net-tor-smtp/) | alpha | SMTP gateway that relays .onion recipients through Tor's SOCKS listener | [🛠️](roles/svc-net-tor-smtp/README.md#quick-setup) | | +| [Translation Gateway](roles/web-svc-translate/) | beta | One translation endpoint that picks the engine itself and prefers a reviewed human string | [🌐](https://s.infinito.nexus/code) [🛠️](roles/web-svc-translate/README.md#quick-setup) | [Keycloak](roles/web-app-keycloak/), [LibreTranslate Engine](roles/svc-ai-libretranslate-engine/), [LiteLLM Gateway](roles/svc-ai-litellm/), [LTEngine](roles/svc-ai-ltengine/), [Ollama](roles/svc-ai-ollama/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [System One](roles/svc-ai-s1/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/), [Weblate](roles/web-app-weblate/) | | [Typesense](roles/svc-db-typesense/) | beta | Central Dockerized search engine shared by multiple apps with per-consumer scoped API keys | [🌐](https://typesense.org/) [🛠️](roles/svc-db-typesense/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Tor Onion Service](roles/svc-net-tor/) | | [Unbound](roles/svc-dns-unbound/) | beta | Central recursive/forwarding DNSSEC resolver on a static IP for consumer stacks to target as their DNS | [🌐](https://nlnetlabs.nl/projects/unbound/about/) [🛠️](roles/svc-dns-unbound/README.md#quick-setup) | | | [Universal Logout](roles/web-svc-logout/) | beta | Central logout proxy for single-action sign-out across all OIDC-connected apps, with Nginx snippets and a live status UI | [🛠️](roles/web-svc-logout/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [web-app-pihole](roles/web-app-pihole/) | beta | Network-wide DNS sinkhole for blocking ads and trackers across all devices, with DHCP and a web admin dashboard | [🌐](https://pi-hole.net/) [🎬](https://youtu.be/KBXTnrD_Zs4) [🛠️](roles/web-app-pihole/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Content Delivery Network](roles/web-svc-cdn/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Weblate](roles/web-app-weblate/) | beta | Translation memory, glossaries and the human review workflow behind the translation gateway | [🌐](https://weblate.org/) [🛠️](roles/web-app-weblate/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Translation Gateway](roles/web-svc-translate/), [Universal Logout](roles/web-svc-logout/) | | [Wireguard](roles/svc-net-wireguard-core/) | alpha | Host-level WireGuard VPN service with wg-quick config, package install, and IP-forwarding sysctl for secure tunnels | [🌐](https://www.wireguard.com/) [🛠️](roles/svc-net-wireguard-core/README.md#quick-setup) | | | [Wireguard Client](roles/svc-net-wireguard-plain/) | alpha | Client-side WireGuard VPN role for stable connectivity with a systemd service and script that tune interface MTU | [🌐](https://www.wireguard.com/) [🛠️](roles/svc-net-wireguard-plain/README.md#quick-setup) | | | [WireGuard Client behind NAT](roles/svc-net-wireguard-firewalled/) | alpha | Networking helper for iptables forwarding and NAT masquerading of a WireGuard client running behind NAT or a firewall | [🛠️](roles/svc-net-wireguard-firewalled/README.md#quick-setup) | | -| [WordPress](roles/web-app-wordpress/) | beta | Content management system for building and managing websites, blogs, and multisite networks with themes and plugins | [🌐](https://wordpress.org/) [🎬](https://www.youtube.com/watch?v=QU8GFqvXRxE) [🛠️](roles/web-app-wordpress/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Discourse](roles/web-app-discourse/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenClaw](roles/web-app-openclaw/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [WordPress](roles/web-app-wordpress/) | beta | Content management system for building and managing websites, blogs, and multisite networks with themes and plugins | [🌐](https://wordpress.org/) [🎬](https://www.youtube.com/watch?v=QU8GFqvXRxE) [🛠️](roles/web-app-wordpress/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [BigBlueButton](roles/web-app-bigbluebutton/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Discourse](roles/web-app-discourse/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Listmonk](roles/web-app-listmonk/), [LiteLLM Gateway](roles/svc-ai-litellm/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenClaw](roles/web-app-openclaw/), [PeerTube](roles/web-app-peertube/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Translation Gateway](roles/web-svc-translate/), [Universal Logout](roles/web-svc-logout/) | | [World Wide Web](roles/web-opt-rdr-www/) | beta | NGINX server blocks that permanently redirect www subdomains to their non-www apex domains | [🛠️](roles/web-opt-rdr-www/README.md#quick-setup) | [Tor Onion Service](roles/svc-net-tor/) | | [XMPP](roles/web-svc-xmpp/) | beta | Placeholder stub role for a future XMPP server/container deployment, implementation pending | [🌐](https://ejabberd.im/) [🛠️](roles/web-svc-xmpp/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Keycloak](roles/web-app-keycloak/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | -| [XWiki](roles/web-app-xwiki/) | beta | Self-hosted enterprise wiki and knowledge-management platform with collaboration and app-based extensibility | [🌐](https://www.xwiki.org/) [🎬](https://www.youtube.com/watch?v=8DePx30dh2g) [🛠️](roles/web-app-xwiki/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [XWiki](roles/web-app-xwiki/) | beta | Self-hosted enterprise wiki and knowledge-management platform with collaboration and app-based extensibility | [🌐](https://www.xwiki.org/) [🎬](https://www.youtube.com/watch?v=8DePx30dh2g) [🛠️](roles/web-app-xwiki/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Matomo](roles/web-app-matomo/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [YOURLS](roles/web-app-yourls/) | beta | Self-hosted URL shortener for creating, tracking, and managing short links with built-in click analytics | [🌐](https://yourls.org/) [🛠️](roles/web-app-yourls/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Zammad](roles/web-app-zammad/) | beta | Open-source helpdesk and ticketing system for handling customer requests via email, chat, phone, and web | [🌐](https://zammad.org/) [🎬](https://www.youtube.com/watch?v=K30V0Uimr-s) [🛠️](roles/web-app-zammad/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Memcached](roles/svc-db-memcached/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | +| [Zammad](roles/web-app-zammad/) | beta | Open-source helpdesk and ticketing system for handling customer requests via email, chat, phone, and web | [🌐](https://zammad.org/) [🎬](https://www.youtube.com/watch?v=K30V0Uimr-s) [🛠️](roles/web-app-zammad/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Matomo](roles/web-app-matomo/), [Memcached](roles/svc-db-memcached/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-svc-seaweedfs/), [Stalwart Mail Server](roles/web-app-stalwart/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | ## Guides 📚 diff --git a/ansible.cfg b/ansible.cfg index 62be0232c0..042543810f 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -9,6 +9,7 @@ retry_files_enabled = False host_key_checking = True deprecation_warnings = True interpreter_python = auto_silent +no_target_syslog = True collections_paths = ./collections:~/.ansible/collections:/usr/share/ansible/collections roles_path = ./roles diff --git a/cli/administration/deploy/development/deploy/cli.py b/cli/administration/deploy/development/deploy/cli.py index 66b86562e4..d32472fff7 100644 --- a/cli/administration/deploy/development/deploy/cli.py +++ b/cli/administration/deploy/development/deploy/cli.py @@ -192,7 +192,7 @@ def handler(args: argparse.Namespace) -> int: f"inv={inv_dir} variants={round_variants} apps={round_deploy_ids}" ) print(f"=== {pass_label} PASS 1 (sync) ===") - rc = run_pass({"VARIANT_INDEX": round_index}) + rc = run_pass({"VARIANT_INDEX": round_index, "PRIMARY_APPS": primary_app_ids}) if rc != 0: return rc @@ -202,7 +202,13 @@ def handler(args: argparse.Namespace) -> int: compose, inventory_dir=inv_dir, round_variants=round_variants ) print(f"=== {pass_label} PASS 2 (async) ===") - rc = run_pass({"ASYNC_ENABLED": True, "VARIANT_INDEX": round_index}) + rc = run_pass( + { + "ASYNC_ENABLED": True, + "VARIANT_INDEX": round_index, + "PRIMARY_APPS": primary_app_ids, + } + ) if rc != 0: return rc diff --git a/cli/administration/recover/recoverers.py b/cli/administration/recover/recoverers.py index 47f3dd737f..e7c35efc3c 100644 --- a/cli/administration/recover/recoverers.py +++ b/cli/administration/recover/recoverers.py @@ -12,6 +12,7 @@ dispatches to one Recoverer via :data:`RECOVERERS`. from __future__ import annotations +import sys from abc import ABC, abstractmethod from cli.administration.recover import paths @@ -83,7 +84,7 @@ class Recoverer(ABC): if not service_backup and self.no_backup_flag: argv.append("--no-safety-backup") argv += self.extra_flags(passphrase_stdin=passphrase_stdin) - return ["python3", self.script(), *argv] + return [sys.executable, self.script(), *argv] class DeviceRecoverer(Recoverer): diff --git a/cli/meta/roles/applications/complexity/render.py b/cli/meta/roles/applications/complexity/render.py index 96c4702a55..0c273342bd 100644 --- a/cli/meta/roles/applications/complexity/render.py +++ b/cli/meta/roles/applications/complexity/render.py @@ -137,7 +137,7 @@ _LIFECYCLE_DOC: dict[str, str] = { "rc": "burn-in on production, intends stable", "stable": "shipped in a release without a hotfix", "maintenance": "stable coverage, feature-frozen", - "deprecated": "kept for compatibility, do not adopt", + "deprecated": "kept for compatibility, untested, do not adopt", "eol": "end of life: shipped but not tested or maintained", } diff --git a/compose.yml b/compose.yml index 92dad0541c..e902b35427 100644 --- a/compose.yml +++ b/compose.yml @@ -88,6 +88,8 @@ services: INFINITO_IMAGE_TAG: ${INFINITO_IMAGE_TAG} INFINITO_IP4: ${INFINITO_IP4} INFINITO_IP6: ${INFINITO_IP6} + INFINITO_MAIL_PRODUCTION_HOST: ${INFINITO_MAIL_PRODUCTION_HOST} + INFINITO_MAIL_PRODUCTION_RESOLVER: ${INFINITO_MAIL_PRODUCTION_RESOLVER} INFINITO_PLAYWRIGHT_KEEP: ${INFINITO_PLAYWRIGHT_KEEP} INFINITO_PLAYWRIGHT_REPORTS_BASE_DIR: ${INFINITO_PLAYWRIGHT_REPORTS_BASE_DIR} INFINITO_SRC_DIR: ${INFINITO_SRC_DIR} diff --git a/compose/swarm/playbook.yml b/compose/swarm/playbook.yml index 6be12b1fb1..007e222128 100644 --- a/compose/swarm/playbook.yml +++ b/compose/swarm/playbook.yml @@ -27,7 +27,7 @@ lab_systemd_timeout: "{{ lookup('env', 'SWARM_PILOT_SYSTEMD_TIMEOUT') | default('180', true) }}" lab_is_mgr: "{{ inventory_hostname == lab_mgr }}" lab_is_worker: "{{ inventory_hostname in [lab_wrk1, lab_wrk2] }}" - lab_dns_node: "{{ lab_is_mgr or lab_is_worker }}" + lab_dns_node: "{{ lab_is_mgr or lab_is_worker or inventory_hostname == lab_nfs }}" lab_domain_target: "{{ lookup('env', 'INFINITO_SWARM_MGR_IP') }}" lab_filesystem: "{{ lookup('env', 'INFINITO_DOCKER_FILESYSTEM') }}" lab_filesystem_required: "{{ lookup('env', 'INFINITO_DOCKER_FILESYSTEM_REQUIRED') | default('false', true) }}" diff --git a/default.env b/default.env index 478fddf66c..2ad070b85a 100644 --- a/default.env +++ b/default.env @@ -17,7 +17,7 @@ INFINITO_ALIAS_REPOSITORY=https://github.com/kevinveenbirkenbach/alias # Requirements file scripts/install/ansible.sh tries first, `galaxy` or `git`; the other one stays the fallback. -INFINITO_ANSIBLE_COLLECTIONS_SOURCE=galaxy +INFINITO_ANSIBLE_COLLECTIONS_SOURCE=git # Anthropic API key the development inventory feeds into API.anthropic.api_key; empty leaves the anthropic routes unpublished. INFINITO_API_ANTHROPIC_KEY="" @@ -223,6 +223,9 @@ INFINITO_INSTANCE=0 # Repo-relative path of the PHP/Ruby unit-suite toolchain install script. INFINITO_INTERPRETERS_INSTALL_SCRIPT=scripts/install/interpreters.sh +# Optional JSON object merged into the generated dev-inventory vars. +INFINITO_INVENTORY_EXTRA_VARS= + # Repo-relative path to the Ansible vars file `infinito administration inventory provision --vars-file <...>` consumes. INFINITO_INVENTORY_VARS_FILE=inventories/development/default.yml @@ -247,6 +250,13 @@ INFINITO_LIMIT_HOST=localhost # Where scripts/lint/wrapper.sh executes 'make lint*' / 'make autoformat': host | docker. INFINITO_LINT_RUNNER=docker +# Public FQDN of a reachable mail server; when set, the provider asserts its live DNS/TLS/SMTP hard facts. Empty disables the check. +INFINITO_MAIL_PRODUCTION_HOST= + +# Resolver the production mail-facts check queries (e.g. 1.1.1.1 for the public view); empty uses /etc/resolv.conf. +INFINITO_MAIL_PRODUCTION_RESOLVER= + + # memswap_limit cap applied to the infinito container; 0 means "unlimited". INFINITO_MEMSWAP_LIMIT=0 @@ -316,6 +326,9 @@ INFINITO_RUN_FLAGS= # Git repository holding the agent skills (skills-lock.json + make project target); developers override this with their own repo. INFINITO_SKILLS_REPOSITORY=https://github.com/kevinveenbirkenbach/skills +# Skip the in-container entry.sh recompile during local deploys (venv prebuilt). +INFINITO_SKIP_COMPILE=false + # Absolute path to the bind-mounted repo root inside the dev container. INFINITO_SRC_DIR=/opt/src/infinito @@ -346,6 +359,9 @@ INFINITO_SWARM_MGR_NAME=swarm-mgr-01 # Fixed lab IP of the NFS-server sidecar. INFINITO_SWARM_NFS_IP=192.168.244.13 +# NFSv4 minor versions the swarm-lab ganesha offers; pin to "0" on hosts whose kernel NFS client breaks ganesha 4.1+ sessions. +INFINITO_SWARM_NFS_MINOR_VERSIONS="0, 1, 2" + # NFS-server sidecar base name in the act-swarm-test cluster. INFINITO_SWARM_NFS_NAME=nfs-server diff --git a/docs/architecture/decisions/0003-mcp-surface-per-provider.md b/docs/architecture/decisions/0003-mcp-surface-per-provider.md index 4457d0d42c..4d09e1b86d 100644 --- a/docs/architecture/decisions/0003-mcp-surface-per-provider.md +++ b/docs/architecture/decisions/0003-mcp-surface-per-provider.md @@ -250,11 +250,11 @@ An upstream path exists or is claimed, and something concrete stops it here. Eac | `web-app-penpot` | `interactive_browser_session` | The server needs an active browser tab and plugin connection, and can execute powerful design-context operations. | | `web-app-shopware` | `version_unverified` | The pinned 6.7.8.2 predates the experimental native `/api/_mcp` server. | -### adapter_candidate (38) +### adapter_candidate (39) An adapter could reach these, and none has been curated yet. The five that used to sit under "current MCP metadata requiring revalidation" are here because that revalidation found no shipped surface to revalidate. -`svc-db-typesense`, `web-app-akaunting`, `web-app-bigbluebutton`, `web-app-bluesky`, `web-app-bookwyrm`, `web-app-bridgy-fed`, `web-app-confluence`, `web-app-decidim`, `web-app-espocrm`, `web-app-friendica`, `web-app-funkwhale`, `web-app-jira`, `web-app-jitsi`, `web-app-joomla`, `web-app-kix`, `web-app-magento`, `web-app-mailu`, `web-app-mastodon`, `web-app-matrix`, `web-app-mediawiki`, `web-app-minio`, `web-app-mobilizon`, `web-app-opencloud`, `web-app-openproject`, `web-app-opentalk`, `web-app-peertube`, `web-app-pihole`, `web-app-pixelfed`, `web-app-postmarks`, `web-svc-seaweedfs`, `web-app-semaphore`, `web-app-socialhome`, `web-app-suitecrm`, `web-app-taiga`, `web-app-xwiki`, `web-app-yourls`, `web-svc-api`, `web-svc-xmpp`. +`svc-db-typesense`, `web-app-akaunting`, `web-app-bigbluebutton`, `web-app-bluesky`, `web-app-bookwyrm`, `web-app-bridgy-fed`, `web-app-confluence`, `web-app-decidim`, `web-app-espocrm`, `web-app-friendica`, `web-app-funkwhale`, `web-app-jira`, `web-app-jitsi`, `web-app-joomla`, `web-app-kix`, `web-app-magento`, `web-app-mailu`, `web-app-mastodon`, `web-app-matrix`, `web-app-mediawiki`, `web-app-minio`, `web-app-mobilizon`, `web-app-opencloud`, `web-app-openproject`, `web-app-opentalk`, `web-app-peertube`, `web-app-pihole`, `web-app-pixelfed`, `web-app-postmarks`, `web-svc-seaweedfs`, `web-app-semaphore`, `web-app-socialhome`, `web-app-stalwart`, `web-app-suitecrm`, `web-app-taiga`, `web-app-xwiki`, `web-app-yourls`, `web-svc-api`, `web-svc-xmpp`. Each candidate starts with three to five named read tools rather than a generated copy of an API, and the first investigation uses the starter contract recorded in [Adapter starter contracts](#adapter-starter-contracts). If the exact pinned version cannot implement the named contract through a stable authenticated API, the role becomes `blocked` with the observed reason rather than receiving a generic substitute. @@ -270,7 +270,7 @@ Each candidate starts with three to five named read tools rather than a generate These are reached through the owning document application's MCP boundary, such as Nextcloud or OpenCloud, rather than receiving a second independent service-account tool surface. -### no_surface (99) +### no_surface (112) These roles stay out of shared MCP discovery. A future exception requires a new record with a fixed operation list, dedicated identity, isolation boundary, human approval for mutations, and audit trail. @@ -279,7 +279,7 @@ A future exception requires a new record with a fixed operation list, dedicated These roles operate a workstation, developer toolchain, device, or host package; bridging them would amount to shared shell, filesystem, browser-session, device, or host execution without an application-specific remote identity. `dsk-base` and `user-workstation` provision the login account the other desktop roles write for, which is the host execution boundary itself rather than an application behind it. -**`privileged_control_plane` (27):** `svc-ai-agent-broker`, `svc-ai-jupyter`, `svc-bkp-local-2-device`, `svc-bkp-nfs-2-local`, `svc-bkp-remote-2-local`, `svc-bkp-secrets-2-local`, `svc-bkp-volume-2-local`, `svc-dns-unbound`, `svc-net-firewall`, `svc-net-tor`, `svc-net-wireguard-core`, `svc-net-wireguard-firewalled`, `svc-net-wireguard-plain`, `svc-opt-keyboard-color`, `svc-opt-ssd-hdd`, `svc-opt-swapfile`, `svc-prx-openresty`, `svc-registry-cache`, `svc-registry-docker`, `svc-runner`, `svc-storage-nfs-client`, `svc-storage-nfs-server`, `svc-swarm-manager`, `svc-swarm-node`, `svc-virt-kata`, `update`, `web-app-openbao`. +**`privileged_control_plane` (28):** `svc-ai-agent-broker`, `svc-ai-jupyter`, `svc-bkp-local-2-device`, `svc-bkp-nfs-2-local`, `svc-bkp-remote-2-local`, `svc-bkp-secrets-2-local`, `svc-bkp-volume-2-local`, `svc-dns-unbound`, `svc-net-firewall`, `svc-net-tor`, `svc-net-tor-smtp`, `svc-net-wireguard-core`, `svc-net-wireguard-firewalled`, `svc-net-wireguard-plain`, `svc-opt-keyboard-color`, `svc-opt-ssd-hdd`, `svc-opt-swapfile`, `svc-prx-openresty`, `svc-registry-cache`, `svc-registry-docker`, `svc-runner`, `svc-storage-nfs-client`, `svc-storage-nfs-server`, `svc-swarm-manager`, `svc-swarm-node`, `svc-virt-kata`, `update`, `web-app-openbao`. These roles can recover secrets, alter routing or host state, run code, change deployment state, or reach storage/control-plane sockets; they require a separate audited operations gateway and human approval rather than general-purpose application MCP. **`shared_engine_isolation` (4):** `svc-db-memcached`, `svc-db-openldap`, `svc-db-postgres`, `svc-db-rabbitmq`. diff --git a/docs/contributing/actions/release.md b/docs/contributing/actions/release.md index 4ba9ada29a..90fb4feb51 100644 --- a/docs/contributing/actions/release.md +++ b/docs/contributing/actions/release.md @@ -41,7 +41,7 @@ A higher count is fine for short bursts of releases, but the active changelog SH Pass `--dry-run` to preview without writing. 3. Run `make test` locally and confirm everything is green. -4. Commit the changelog and any archive additions in a single commit with subject `Release version ..` (matching the historical convention). +4. Commit the changelog and any archive additions in a single commit with subject `Release version ..` (matching the historical convention). A push to `main` whose head commit carries that subject cancels the running `main` pipeline, and [test_release_tag_commit_subject.py](../../../tests/integration/meta/workflows/test_release_tag_commit_subject.py) fails for every tag from `v13.0.0` on that sits on another subject. 5. Tag the commit: `git tag -a v.. -m "Release version .."`. 6. Push the branch and the tag with [git-sign-push](https://github.com/kevinveenbirkenbach/git-maintainer-tools), running it outside the sandbox. diff --git a/docs/contributing/design/role/services/email.md b/docs/contributing/design/role/services/email.md index b73d65e70d..aff9f0f663 100644 --- a/docs/contributing/design/role/services/email.md +++ b/docs/contributing/design/role/services/email.md @@ -89,10 +89,10 @@ Role templates and tasks that consume the lookup MUST follow these rules: - For `vars/main.yml`, contributors MUST define a single holder variable (`_EMAIL: "{{ lookup('email', application_id) }}"`) and reference its keys. Re-calling the lookup on every reference would re-run the resolution. - Non-app consumers (any `sys-*` role) MUST call `lookup('email')` without an `application_id` so they never trigger a per-app merge. -## Mailu Dependency Detection 🚨 +## Mail Provider Dependency Detection 🚨 -Roles that call `lookup('email', ...)` are treated as dependents of Mailu. -The integration check [test_mailu_dependency.py](../../../../../tests/integration/infrastructure/services/test_mailu_dependency.py) scans every role for the `lookup('email'` pattern and fails if such a role does not declare `services.email` with `enabled: true` and `shared: true` at the file root of its `meta/services.yml`. +Roles that call `lookup('email', ...)` are treated as dependents of the mail provider (`web-app-stalwart`). +The integration check [test_mail_provider_dependency.py](../../../../../tests/integration/infrastructure/services/test_mail_provider_dependency.py) scans every role for the `lookup('email'` pattern and fails if such a role does not declare `services.email` with `enabled: true` and `shared: true` at the file root of its `meta/services.yml`. A new email consumer MUST declare that service block in its role's `meta/services.yml`. ## See Also 🔗 diff --git a/docs/contributing/design/role/services/layout.md b/docs/contributing/design/role/services/layout.md index f8f8368f16..b1aa17287d 100644 --- a/docs/contributing/design/role/services/layout.md +++ b/docs/contributing/design/role/services/layout.md @@ -342,7 +342,7 @@ web: ### Allowed `lifecycle` Values -`planned`, `pre-alpha`, `alpha`, `beta`, `stable`, `deprecated`. Unknown values fail the lint. +The stages listed in [lifecycle.md](lifecycle.md#stages-). Unknown values fail the lint. ### `run_after` Rules diff --git a/docs/contributing/design/role/services/lifecycle.md b/docs/contributing/design/role/services/lifecycle.md index 36ef33dc42..094eb8c8e9 100644 --- a/docs/contributing/design/role/services/lifecycle.md +++ b/docs/contributing/design/role/services/lifecycle.md @@ -19,7 +19,7 @@ pre-alpha (early scaffolding, not yet stable enough to test) │ rc │ │ stable │ └─ maintenance ─┘ -deprecated (kept for compatibility, do not adopt for new deploys) +deprecated (kept for compatibility, untested, do not adopt for new deploys) eol (end of life: shipped but not tested or maintained) ``` @@ -95,12 +95,12 @@ Same test coverage as `stable`. ### deprecated ⚠️ -The role still ships and still passes its tests, but operators MUST migrate away from it. +The role still ships, but operators MUST migrate away from it and the project no longer tests it. A `deprecated` role: - MUST be tagged with a "Deprecated" banner in its `README.md` pointing at the successor. -- MUST keep working until removed (the test suite still covers it). +- MUST NOT be exercised by the matrix-deploy, and a failure in it MUST NOT block any release. - SHOULD be removed within a small number of releases. ### eol 🪦 @@ -128,7 +128,8 @@ The matrix-deploy + Playwright pipeline (see [variants.md](../../variants.md) an A regression in any tested-envelope role blocks the merge that introduced it. Stages outside the envelope (`planned`, `pre-alpha`, `deprecated`, `eol`) MAY skip the matrix-deploy gate. -CI MAY still exercise them on a best-effort basis but failures MUST NOT block unrelated work. +CI MAY still exercise `planned`, `pre-alpha` and `eol` roles on a best-effort basis, but failures MUST NOT block unrelated work. +`deprecated` roles are not exercised at all. Promotion from `eol` back into the tested envelope is allowed but requires meeting the `alpha` criteria from scratch. Promoting also reverses the storage steps, in this order: change `lifecycle`, set the primary entity's `enabled` and `shared` to the values the role needs, restore the `bond` the primary carried before (`test_disabled_service_bond` is what required dropping it, because the resource model never reads a key on an entry stored off, so the number described a cost nobody paid), restore the consumer flags from their literal `false` to the dynamic form and drop the `# nocheck: dynamic-flag` markers that literal needed, re-add the variant pins the EOL ban removed, then re-run the matrix-deploy. diff --git a/docs/contributing/tools/agents/claude/sandbox.md b/docs/contributing/tools/agents/claude/sandbox.md index 939b7c36ae..f03562cd90 100644 --- a/docs/contributing/tools/agents/claude/sandbox.md +++ b/docs/contributing/tools/agents/claude/sandbox.md @@ -83,7 +83,7 @@ The `sandbox.network` block governs outbound and local connectivity from inside | Field | Current value | Effect | |---|---|---| -| `allowedDomains` | List of domain patterns the agent or its tooling needs to reach. Apex domains are listed bare (`pypi.org`, `github.com`, `ghcr.io`, `letsencrypt.org`, `monogramm.io`, `cybermaster.space`, `infinito.nexus`, `infinito.test`); subdomain families use the `*.parent.tld` form (`*.pythonhosted.org`, `*.ansible.com`, `*.amazonaws.com`, `*.github.com`, `*.githubusercontent.com`, `*.npmjs.org`, `*.docker.com`, `*.gitea.com`, `*.hcaptcha.com`, `*.gstatic.com`, `*.taiga.io`, `*.infinito.nexus`). | Subdomain wildcards (`*.example.com`) are honored, but a bare `*` is not; sandboxed network access is enforced per pattern, and any unmatched host triggers an interactive `SandboxNetworkAccess` prompt. Add a new pattern here when introducing tooling that fetches from a new origin; egress is still bounded by any host-level firewall on top. | +| `allowedDomains` | List of domain patterns the agent or its tooling needs to reach. Apex domains are listed bare (`pypi.org`, `github.com`, `ghcr.io`, `letsencrypt.org`, `monogramm.io`, `cybermaster.space`, `infinito.nexus`, `infinito.test`); subdomain families use the `*.parent.tld` form (`*.pythonhosted.org`, `*.ansible.com`, `*.amazonaws.com`, `*.github.com`, `*.githubusercontent.com`, `*.npmjs.org`, `*.docker.com`, `*.gitea.com`, `*.hcaptcha.com`, `*.gstatic.com`, `*.taiga.io`, `*.infinito.nexus`, `*.infinito.test`). | Subdomain wildcards (`*.example.com`) are honored, but a bare `*` is not; sandboxed network access is enforced per pattern, and any unmatched host triggers an interactive `SandboxNetworkAccess` prompt. Add a new pattern here when introducing tooling that fetches from a new origin; egress is still bounded by any host-level firewall on top. | | `allowAllUnixSockets` | `true` | The sandbox can connect to any Unix-domain socket on the host. Required so commands like `docker`, `systemctl`, and language servers continue to work; these all communicate over Unix sockets (e.g. `/var/run/docker.sock`). | | `allowLocalBinding` | `true` | The agent MAY bind listening sockets on `localhost` (e.g. `make compose-up`, `python -m http.server`, dev servers). Required for any local end-to-end testing. | diff --git a/docs/contributing/tools/github/actions/configuration.md b/docs/contributing/tools/github/actions/configuration.md index 5703ea18b5..18fd9633bd 100644 --- a/docs/contributing/tools/github/actions/configuration.md +++ b/docs/contributing/tools/github/actions/configuration.md @@ -16,6 +16,8 @@ Repository variables are set under **Settings → Secrets and variables → Acti | `API_OPENAI_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | The gateway publishes no `openai/*` model, and the CLI test asserts their absence | An OpenAI key, to deploy and test the route | | `API_ANTHROPIC_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | The gateway publishes no `anthropic/*` model | An Anthropic key | | `API_OPENROUTER_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | The gateway publishes no `openrouter/auto` model | An OpenRouter key | +| `INFINITO_MAIL_PRODUCTION_HOST` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | The read-only production mail assertions are skipped, because the task that runs them is gated on this host being set | The public hostname of a deployed mail server, to assert its MX, SPF, DKIM, DMARC and PTR records and its SMTP greeting after a deploy; nothing is sent and nothing is mutated | +| `INFINITO_MAIL_PRODUCTION_RESOLVER` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | Those assertions resolve through the runner's own resolver | A DNS server address, to resolve the records against it instead; ignored while `INFINITO_MAIL_PRODUCTION_HOST` is unset | ## Provider keys: variable or secret? 🔑 @@ -41,13 +43,13 @@ resolve to empty inside the container without any error. ## Cancelling in-progress runs 🛑 -Not a variable: cancellation is derived from the trigger. Every automatic run on a branch other than `main` is cancelled by a newer run in the same concurrency group; automatic runs on `main` are not, because a half-finished pipeline on the default branch is worse than a queued one. A manual dispatch always supersedes, `main` included: someone typed it, so it is the intent that counts, not the ref. +Not a variable: cancellation is derived from the trigger. Every automatic run on a branch other than `main` is cancelled by a newer run in the same concurrency group; automatic runs on `main` are not, because a half-finished pipeline on the default branch is worse than a queued one. The one exception on `main` is a release push: a head commit whose subject starts with `Release version `, the subject pkgmgr writes for every release commit, cancels the run in progress so the release starts at once. A manual dispatch always supersedes, `main` included: someone typed it, so it is the intent that counts, not the ref. ```yaml -cancel-in-progress: ${{ github.ref_name != 'main' }} +cancel-in-progress: ${{ github.ref_name != 'main' || (github.ref_name == 'main' && startsWith(github.event.head_commit.message, 'Release version ')) }} ``` -`entry-push-latest.yml` declares exactly that expression, and it fires only on `push`, where `github.ref_name` is always the pushed branch. [entry-cancel-superseded.yml](../../../../../.github/workflows/entry-cancel-superseded.yml) carries no concurrency group of its own — it is the API fallback for that group — but its push job repeats the same predicate as a job `if`, so it never touches `main` either. +`entry-push-latest.yml` declares exactly that expression, and it fires only on `push`, where `github.ref_name` is always the pushed branch. [entry-cancel-superseded.yml](../../../../../.github/workflows/entry-cancel-superseded.yml) carries no concurrency group of its own — it is the API fallback for that group — but its push job repeats the same predicate as a job `if`. It does not listen on `main`, so a release push there relies on the group alone. > ⚠️ **Do not paste that expression into a workflow that listens on `pull_request_target`.** There `github.ref` is the *base* branch, so `github.ref_name` is literally `main` and the rule collapses to a constant `false` — every pull request would stop cancelling its predecessor. `entry-pr-change-orchestrate.yml` therefore declares `cancel-in-progress: true` outright. diff --git a/group_vars/all/00_general.yml b/group_vars/all/00_general.yml index 7d084f8909..e044287449 100644 --- a/group_vars/all/00_general.yml +++ b/group_vars/all/00_general.yml @@ -40,6 +40,11 @@ DNS_PROVIDER: "localhost" # The DNS Provider\Registrar for the domain -> Legit v HOSTING_PROVIDER: "localhost" # Provider which hosts the server -> hetzner +MAIL_PROVIDER: "web-app-stalwart" # Active mail provider role. Set to web-app-mailu to use Mailu instead of Stalwart. + +MAIL_PRODUCTION_HOST: "" # Public FQDN of a reachable mail server; when set, the provider asserts its live DNS/TLS/SMTP hard facts. +MAIL_PRODUCTION_RESOLVER: "" # Resolver those assertions query; empty uses the host's /etc/resolv.conf. + DOCKER_IN_CONTAINER: "{{ (ansible_facts['env']['container'] | default('')) != '' }}" DOCKER_RESTART_POLICY: "unless-stopped" # Default restart parameter for docker containers DOCKER_WHITELISTET_ANON_VOLUMES: [] # Volumes which should be ignored during docker anonymous health check @@ -54,6 +59,8 @@ ASYNC_POLL: "{{ 0 if ASYNC_ENABLED | bool else 10 }}" # Don't wait for task APPLICATIONS_WHITELIST: [] # nocheck: unused-var +PRIMARY_APPS: [] # Apps this deploy was asked for; the rest of the round is their dependency closure. + RBAC: GROUP: NAME: "/roles" # Name of the group which holds the RBAC roles diff --git a/group_vars/all/05_paths.yml b/group_vars/all/05_paths.yml index 1ff25572d2..b5a7f5fdd1 100644 --- a/group_vars/all/05_paths.yml +++ b/group_vars/all/05_paths.yml @@ -29,6 +29,8 @@ DIR_SECRETS: "{{ [DIR_VAR_LIB, 'secrets'] | path_join }}" FILE_TOKENS: "{{ [DIR_SECRETS, 'tokens.yml'] | path_join }}" FILE_DATABASE_SECRETS: "{{ [DIR_SECRETS, 'databases.csv'] | path_join }}" +DIR_METRICS_TEXTFILE: "{{ [DIR_VAR_LIB, 'metrics'] | path_join }}" + DIR_RESCUE_DIAGNOSTICS: "/tmp/infinito-rescue-diagnostics" FILE_DNS53_SAMPLER_LOG: "/var/log/infinito/dns53-sampler.log" diff --git a/group_vars/all/14_about.yml b/group_vars/all/14_about.yml index ac21c5cd04..f6c5b976eb 100644 --- a/group_vars/all/14_about.yml +++ b/group_vars/all/14_about.yml @@ -20,7 +20,7 @@ defaults_service_provider: web-app-bluesky: >- {{ ('@' ~ lookup('users', 'contact').username ~ '.' ~ lookup('domains', 'web-app-bluesky').api) if 'web-app-bluesky' in group_names else '' }} - email: "{{ lookup('users', 'contact').username ~ '@' ~ DOMAIN_PRIMARY if 'web-app-mailu' in group_names else 'contact@infinito.nexus' }}" + email: "{{ lookup('users', 'contact').username ~ '@' ~ DOMAIN_PRIMARY if MAIL_PROVIDER in group_names else 'contact@infinito.nexus' }}" mastodon: "{{ '@' ~ lookup('users', 'contact').username ~ '@' ~ lookup('domain','web-app-mastodon') if 'web-app-mastodon' in group_names else '@infinito@microblog.infinito.nexus' }}" matrix: "{{ '@' ~ lookup('users', 'contact').username ~ ':' ~ lookup('config','web-app-matrix', 'services.matrix.server_name') if 'web-app-matrix' in group_names diff --git a/inventories/development/default.yml b/inventories/development/default.yml index 24e8a8ae35..f8007324cf 100644 --- a/inventories/development/default.yml +++ b/inventories/development/default.yml @@ -23,6 +23,8 @@ INFINITO_IMAGE_TAG: "{{ lookup('env', 'INFINITO_IMAGE_TAG') | default('latest', INFINITO_GHCR_MIRROR_PREFIX: "{{ lookup('env', 'INFINITO_GHCR_MIRROR_PREFIX') | default('', true) }}" CA_CONTAINER_ENABLED: "{{ lookup('env', 'INFINITO_CONTAINER_CA_ENABLED') | bool }}" test_e2e_playwright_keep: "{{ lookup('env', 'INFINITO_PLAYWRIGHT_KEEP') | bool }}" +MAIL_PRODUCTION_HOST: "{{ lookup('env', 'INFINITO_MAIL_PRODUCTION_HOST') | default('', true) }}" +MAIL_PRODUCTION_RESOLVER: "{{ lookup('env', 'INFINITO_MAIL_PRODUCTION_RESOLVER') | default('', true) }}" SYS_TIMEOUT_CONTAINER_EXEC: "60min" networks: internet: @@ -46,6 +48,7 @@ API: OIDC_OVERRIDE: CLIENT: SECRET: VERYUNSAFESAMPLEPASSWORD1234 +PLAYWRIGHT_MAPACHE_PASSWORD: "VeryUnsecureMapache!1" users: administrator: authorized_keys: @@ -61,6 +64,7 @@ users: uid: 10001 gid: 10001 + accounts: ["mailbox", "identity"] roles: [] tokens: {} authorized_keys: [] diff --git a/inventories/development/swarm.yml b/inventories/development/swarm.yml index 0125a89ceb..bbd3eec84a 100644 --- a/inventories/development/swarm.yml +++ b/inventories/development/swarm.yml @@ -22,6 +22,11 @@ storage: SVC_STORAGE_NFS_SERVER_STATE_ALLOWED_CIDRS: - "192.168.244.0/24" # nocheck: hardcoded-subnet (act swarm-lab subnet; nodes + controller mount the fsid=0 state export from it) +SVC_STORAGE_NFS_SERVER_MINOR_VERSIONS: "{{ lookup('env', 'INFINITO_SWARM_NFS_MINOR_VERSIONS') | default('0, 1, 2', true) }}" # nocheck: swarm-only inventory; evaluated on the CI runner where load.sh exports it, never inside the compose DiD + +MAIL_PRODUCTION_HOST: "{{ lookup('env', 'INFINITO_MAIL_PRODUCTION_HOST') | default('', true) }}" +MAIL_PRODUCTION_RESOLVER: "{{ lookup('env', 'INFINITO_MAIL_PRODUCTION_RESOLVER') | default('', true) }}" + swarm: network: encryption: true diff --git a/library/keycloak_kcadm_update.py b/library/keycloak_kcadm_update.py index d2ab477e03..3913e0ec17 100755 --- a/library/keycloak_kcadm_update.py +++ b/library/keycloak_kcadm_update.py @@ -9,6 +9,7 @@ from copy import deepcopy from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.kcadm_json import json_from_noisy_stdout +from ansible.module_utils.keycloak_scopes import converge_client_scope_lists NO_SUCH_CONTAINER_RE = re.compile(r"no such container", re.IGNORECASE) DEAD_CID_MAX_RETRIES = 80 @@ -469,6 +470,11 @@ def run_module(): payload=desired_obj, ) + if object_kind == "client": + converge_client_scope_lists( + run_kcadm, module, object_id, desired, realm, kcadm_exec + ) + result["changed"] = True result["result"] = desired_obj module.exit_json(**result) diff --git a/package-lock.json b/package-lock.json index 9a924a5fe1..b8108859b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -627,9 +627,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.9", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", - "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 1b3bf5bc45..d7d0852297 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,6 @@ "stylelint": "^17.15.0" }, "overrides": { - "brace-expansion": "^5.0.9" + "brace-expansion": "^5.0.12" } } diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD index 08a16cc965..8b6f777599 100644 --- a/packaging/arch/PKGBUILD +++ b/packaging/arch/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: Kevin Veen-Birkenbach pkgname=infinito-nexus -pkgver=14.2.0 +pkgver=15.0.0 pkgrel=1 pkgdesc="Meta package for Infinito.Nexus host dependencies (dev + CI)." arch=('any') diff --git a/packaging/debian/changelog b/packaging/debian/changelog index 86c653a99b..9b775eb6fe 100644 --- a/packaging/debian/changelog +++ b/packaging/debian/changelog @@ -1,3 +1,146 @@ +infinito-nexus (15.0.0-1) unstable; urgency=medium + + **Breaking Changes** + + * **Stalwart replaces Mailu as the default mail provider.** *MAIL_PROVIDER* now defaults to + *web-app-stalwart*, and every application switches its mail settings on only while the + configured provider is deployed. An inventory that keeps Mailu must set + *MAIL_PROVIDER: web-app-mailu*, otherwise its applications stop sending mail. Mailu is now + *deprecated*: it keeps working until it is removed, but new deployments must not adopt it, + and CI no longer deploys or tests it, the mailbox import included. To move over, deploy + Stalwart next to Mailu and switch on *services.stalwart.migration.import_mailu*: in compose + mode the role imports every inventory account's mail from Mailu over IMAP, keeping + folders, flags and dates, and re-runs skip what is already there. See + [web-app-stalwart](roles/web-app-stalwart/README.md). + + * **PostgreSQL 18 on the official image.** PGDG archived the Debian 11 repository that the + old *postgis/postgis:17-3.5* base installed from, so every build of the shared database + image failed. *svc-db-postgres* and the per-application database now run the official + multi-arch *postgres:18* image, and the data volume mounts at */var/lib/postgresql*. + Existing PostgreSQL 17 clusters are not migrated. Dump every database before you deploy + this release and restore it into the new cluster afterwards. PostGIS is installed only for + a role that declares it. + + * **Major versions arrive with the image updates.** MariaDB moves from 12.3 to 13.0 and + upgrades its system tables on start. Odoo moves from 19.0 to 20.0 and does not migrate an + existing database across major versions on its own, so back up Odoo before you deploy. + XWiki moves from 17.10 to 18.8, and the OpenLDAP base from Debian 12 to 13. + + **For Users** + + * **Stalwart mail server.** The new *web-app-stalwart* role runs SMTP, submission, IMAP, + POP3, JMAP, ManageSieve, CalDAV, CardDAV and WebDAV in one server, with spam filtering, + DKIM and DMARC signing and a web admin interface, plus Roundcube webmail and a ClamAV scan + of incoming mail. Domains, accounts, TLS, Keycloak sign-in, DKIM keys and DNS records are + provisioned through Stalwart's management API on every deploy, in compose and in swarm + mode. Two silent failures are closed: a Keycloak that was still starting could leave + Stalwart without single sign-on for the life of the process, and the server wrote no logs + at all. Both are fixed. See [web-app-stalwart](roles/web-app-stalwart/README.md). + + * **Mail to and from onion addresses.** The new *svc-net-tor-smtp* role accepts mail for + *.onion* recipients and delivers it over Tor. The applications reach an onion mail + provider too: 27 roles now resolve the provider's onion host inside their containers and + skip STARTTLS towards it, since no certificate can exist for an onion name and Tor already + encrypts the link. GitLab's test suite checks that a password reset mail actually arrives. + + * **GitLab migrations no longer run out of locks.** GitLab's database migration failed on + PostgreSQL 18 with "out of shared memory". The shared and the per-application PostgreSQL + now allow 256 locks per transaction. + + * **Backups survive a file truncated mid-copy.** *baudolo* 7.0.2 lets the stopped checksum + copy replace a live pre-copy that a writing container broke by truncating files under + rsync, which used to abort the backup. + + **For Developers** + + * **The mail provider is a service, not a role name.** Stalwart declares *provides: email* + and Mailu *covers: [email]*. The new *mail_provider* lookup returns the configured provider + while it is deployed, otherwise the first deployed role offering *email*, and reads the + cluster-wide groups so swarm workers agree with the manager. The port-collision lint + accepts a collision only inside a registry-declared provider-alternative group, and a new + lint fails every role that reads the *email* lookup without emitting + *container_extra_hosts*. + + * **Deprecated roles are untested.** A role in the *deprecated* lifecycle stage is no longer + deployed or tested by CI and cannot block a release, so Stalwart stopped pulling Mailu into + its deploy round. The allowed lifecycle values now have one source, the stage list in + [lifecycle.md](docs/contributing/design/role/services/lifecycle.md). + + * **Release pushes on main.** A push to *main* whose head commit starts with + *Release version* cancels the running *main* pipeline, so the release starts at once. A + release-tagged commit deploys every chunk instead of stopping at the first failed one, and + still ends red when any chunk fails, which keeps a broken release from being published. A + new integration test requires every release tag from v13.0.0 on to sit on that subject, + and the test job now checks out the full history with tags. + + * **Cancelling CI runs.** Pull request runs fail fast: the first failed row cancels its + siblings in the deploy and workspace matrices, while push and manual runs keep every row. + Identical deploy rows from different runs queue behind each other again, keyed on every + matrix axis. A cancel now also stops jobs that used to run after a failure: + *!cancelled()* replaces *always()*, enforced by lint, and only *done* and + *report-main-failures* still report a cancelled run. CodeRabbit reviews draft pull + requests too. + + * **Onion port probe runs everywhere.** A role with more than one onion-forwarded port wrote + an unquoted port list into *test.env*, so sourcing it left the list empty and the probe + passed without checking anything. The list is quoted now, and a unit test sources the + rendered file in compose and swarm mode. + + * **Agent sandbox.** *git commit --amend* is denied, the host's credential files stay + unreadable inside the sandbox, sandboxed commands may reach local services on 127.0.0.1 + to 127.0.0.12 and localhost, and third-party plugins are no longer switched on for every + contributor. + + * **Ansible collections from Git first.** The automated retune made Git the preferred + source for Ansible collections, with Galaxy as the fallback, after Galaxy's median install + time reached three times Git's. + + * Image and dependency version jumps (net since 14.2.0): + * *svc-db-postgres*: postgis/postgis 17-3.5 to postgres 18 + * *svc-db-mariadb*: 12.3 to 13.0 + * *svc-db-redis*: 8.10.1-alpine to 8.10.2-alpine + * *svc-db-openldap* (Debian base): 12.15-slim to 13.7-slim + * *svc-ai-ollama*: 0.34.2 to 0.35.0 + * *svc-runner* (BuildKit): v0.33.0 to v0.33.1 + * *svc-net-tor-smtp*: newly on python 3.14-slim + * *web-app-stalwart*: newly pinned to Stalwart v0.16.24, Roundcube 1.7.4-apache, ClamAV + 1.5.4 and postgres 18 + * *web-app-dashboard* (port-ui): 2.1.3 to 2.2.2 + * *web-app-erpnext*: v16.35.0 to v16.37.0 + * *web-app-espocrm*: 10.0.8 to 10.0.9 + * *web-app-gitlab*: v19.3.2 to v19.4.1 (all six CNG component images) + * *web-app-joomla*: 6.1.3-php8.3-apache to 6.1.4-php8.3-apache + * *web-app-keycloak*: 26.7.4 to 26.8.0, oauth2-proxy v7.15.4 to v7.15.5 + * *web-app-magento*: OpenSearch 3.8.0 to 3.9.0 + * *web-app-matomo*: 5.13.0 to 5.14.0 + * *web-app-matrix*: Synapse v1.160.0 to v1.162.0, Element v1.12.28 to v1.12.30 + * *web-app-odoo*: 19.0 to 20.0 + * *web-app-peertube*: v8.3.0 to v8.3.1 + * *web-app-seaweedfs*: 4.46 to 4.48 + * *web-app-shopware*: OpenSearch 3.8.0 to 3.9.0 + * *web-app-snipe-it*: v8.7.2-alpine to v8.8.0-alpine + * *web-app-socialhome*: 2026.9.18 to 2026.10.2 + * *web-app-xwiki*: 17.10.13 to 18.8.0 + * *web-app-bluesky* (view): 1.132.0 to 1.134.0 + * *web-app-discourse*: v2026.8.0 to v2026.9.0 + * *web-svc-logout*: universal-logout v1.3.1 to v1.4.1 + * *web-svc-xmpp*: 26.07 to 26.09 + * *pkgmgr*: v1.16.1 to v2.0.2 + * *backup-docker-to-local* (*baudolo*): 7.0.1 to 7.0.2 + * *ruff* (dev): 0.16.7 to 0.16.9 + * *brace-expansion* (dev): ^5.0.9 to ^5.0.12 + + **Contributors** + + * [Alejandro Roman Ibanez](https://github.com/AlejandroRomanIbanez): Stalwart as the default + mail provider in compose and swarm mode, the Tor SMTP gateway, onion mail delivery for + every mail-sending role, the Mailu import and the provider resolution + * [Kevin Veen-Birkenbach](https://veen.world): PostgreSQL 18, release-aware CI cancellation + and chunk gating, the release tag test, the Mailu deprecation and the untested deprecated + stage, the agent sandbox policy, review and version maintenance + + -- Kevin Veen-Birkenbach Sat, 03 Oct 2026 21:21:58 +0200 + infinito-nexus (14.2.0-1) unstable; urgency=medium **For Users** diff --git a/packaging/fedora/infinito-nexus.spec b/packaging/fedora/infinito-nexus.spec index 9c97657c75..d87e2f2e42 100644 --- a/packaging/fedora/infinito-nexus.spec +++ b/packaging/fedora/infinito-nexus.spec @@ -1,5 +1,5 @@ Name: infinito-nexus -Version: 14.2.0 +Version: 15.0.0 Release: 1%{?dist} Summary: Meta package for Infinito.Nexus host dependencies @@ -58,6 +58,146 @@ install -d %{buildroot}%{_docdir}/%{name} %doc %{_docdir}/%{name}/DEPENDENCIES %changelog +* Sat Oct 03 2026 Kevin Veen-Birkenbach - 15.0.0-1 +- **Breaking Changes** + +- * **Stalwart replaces Mailu as the default mail provider.** *MAIL_PROVIDER* now defaults to +- *web-app-stalwart*, and every application switches its mail settings on only while the +- configured provider is deployed. An inventory that keeps Mailu must set +- *MAIL_PROVIDER: web-app-mailu*, otherwise its applications stop sending mail. Mailu is now +- *deprecated*: it keeps working until it is removed, but new deployments must not adopt it, +- and CI no longer deploys or tests it, the mailbox import included. To move over, deploy +- Stalwart next to Mailu and switch on *services.stalwart.migration.import_mailu*: in compose +- mode the role imports every inventory account's mail from Mailu over IMAP, keeping +- folders, flags and dates, and re-runs skip what is already there. See +- [web-app-stalwart](roles/web-app-stalwart/README.md). + +- * **PostgreSQL 18 on the official image.** PGDG archived the Debian 11 repository that the +- old *postgis/postgis:17-3.5* base installed from, so every build of the shared database +- image failed. *svc-db-postgres* and the per-application database now run the official +- multi-arch *postgres:18* image, and the data volume mounts at */var/lib/postgresql*. +- Existing PostgreSQL 17 clusters are not migrated. Dump every database before you deploy +- this release and restore it into the new cluster afterwards. PostGIS is installed only for +- a role that declares it. + +- * **Major versions arrive with the image updates.** MariaDB moves from 12.3 to 13.0 and +- upgrades its system tables on start. Odoo moves from 19.0 to 20.0 and does not migrate an +- existing database across major versions on its own, so back up Odoo before you deploy. +- XWiki moves from 17.10 to 18.8, and the OpenLDAP base from Debian 12 to 13. + +- **For Users** + +- * **Stalwart mail server.** The new *web-app-stalwart* role runs SMTP, submission, IMAP, +- POP3, JMAP, ManageSieve, CalDAV, CardDAV and WebDAV in one server, with spam filtering, +- DKIM and DMARC signing and a web admin interface, plus Roundcube webmail and a ClamAV scan +- of incoming mail. Domains, accounts, TLS, Keycloak sign-in, DKIM keys and DNS records are +- provisioned through Stalwart's management API on every deploy, in compose and in swarm +- mode. Two silent failures are closed: a Keycloak that was still starting could leave +- Stalwart without single sign-on for the life of the process, and the server wrote no logs +- at all. Both are fixed. See [web-app-stalwart](roles/web-app-stalwart/README.md). + +- * **Mail to and from onion addresses.** The new *svc-net-tor-smtp* role accepts mail for +- *.onion* recipients and delivers it over Tor. The applications reach an onion mail +- provider too: 27 roles now resolve the provider's onion host inside their containers and +- skip STARTTLS towards it, since no certificate can exist for an onion name and Tor already +- encrypts the link. GitLab's test suite checks that a password reset mail actually arrives. + +- * **GitLab migrations no longer run out of locks.** GitLab's database migration failed on +- PostgreSQL 18 with "out of shared memory". The shared and the per-application PostgreSQL +- now allow 256 locks per transaction. + +- * **Backups survive a file truncated mid-copy.** *baudolo* 7.0.2 lets the stopped checksum +- copy replace a live pre-copy that a writing container broke by truncating files under +- rsync, which used to abort the backup. + +- **For Developers** + +- * **The mail provider is a service, not a role name.** Stalwart declares *provides: email* +- and Mailu *covers: [email]*. The new *mail_provider* lookup returns the configured provider +- while it is deployed, otherwise the first deployed role offering *email*, and reads the +- cluster-wide groups so swarm workers agree with the manager. The port-collision lint +- accepts a collision only inside a registry-declared provider-alternative group, and a new +- lint fails every role that reads the *email* lookup without emitting +- *container_extra_hosts*. + +- * **Deprecated roles are untested.** A role in the *deprecated* lifecycle stage is no longer +- deployed or tested by CI and cannot block a release, so Stalwart stopped pulling Mailu into +- its deploy round. The allowed lifecycle values now have one source, the stage list in +- [lifecycle.md](docs/contributing/design/role/services/lifecycle.md). + +- * **Release pushes on main.** A push to *main* whose head commit starts with +- *Release version* cancels the running *main* pipeline, so the release starts at once. A +- release-tagged commit deploys every chunk instead of stopping at the first failed one, and +- still ends red when any chunk fails, which keeps a broken release from being published. A +- new integration test requires every release tag from v13.0.0 on to sit on that subject, +- and the test job now checks out the full history with tags. + +- * **Cancelling CI runs.** Pull request runs fail fast: the first failed row cancels its +- siblings in the deploy and workspace matrices, while push and manual runs keep every row. +- Identical deploy rows from different runs queue behind each other again, keyed on every +- matrix axis. A cancel now also stops jobs that used to run after a failure: +- *!cancelled()* replaces *always()*, enforced by lint, and only *done* and +- *report-main-failures* still report a cancelled run. CodeRabbit reviews draft pull +- requests too. + +- * **Onion port probe runs everywhere.** A role with more than one onion-forwarded port wrote +- an unquoted port list into *test.env*, so sourcing it left the list empty and the probe +- passed without checking anything. The list is quoted now, and a unit test sources the +- rendered file in compose and swarm mode. + +- * **Agent sandbox.** *git commit --amend* is denied, the host's credential files stay +- unreadable inside the sandbox, sandboxed commands may reach local services on 127.0.0.1 +- to 127.0.0.12 and localhost, and third-party plugins are no longer switched on for every +- contributor. + +- * **Ansible collections from Git first.** The automated retune made Git the preferred +- source for Ansible collections, with Galaxy as the fallback, after Galaxy's median install +- time reached three times Git's. + +- * Image and dependency version jumps (net since 14.2.0): +- * *svc-db-postgres*: postgis/postgis 17-3.5 to postgres 18 +- * *svc-db-mariadb*: 12.3 to 13.0 +- * *svc-db-redis*: 8.10.1-alpine to 8.10.2-alpine +- * *svc-db-openldap* (Debian base): 12.15-slim to 13.7-slim +- * *svc-ai-ollama*: 0.34.2 to 0.35.0 +- * *svc-runner* (BuildKit): v0.33.0 to v0.33.1 +- * *svc-net-tor-smtp*: newly on python 3.14-slim +- * *web-app-stalwart*: newly pinned to Stalwart v0.16.24, Roundcube 1.7.4-apache, ClamAV +- 1.5.4 and postgres 18 +- * *web-app-dashboard* (port-ui): 2.1.3 to 2.2.2 +- * *web-app-erpnext*: v16.35.0 to v16.37.0 +- * *web-app-espocrm*: 10.0.8 to 10.0.9 +- * *web-app-gitlab*: v19.3.2 to v19.4.1 (all six CNG component images) +- * *web-app-joomla*: 6.1.3-php8.3-apache to 6.1.4-php8.3-apache +- * *web-app-keycloak*: 26.7.4 to 26.8.0, oauth2-proxy v7.15.4 to v7.15.5 +- * *web-app-magento*: OpenSearch 3.8.0 to 3.9.0 +- * *web-app-matomo*: 5.13.0 to 5.14.0 +- * *web-app-matrix*: Synapse v1.160.0 to v1.162.0, Element v1.12.28 to v1.12.30 +- * *web-app-odoo*: 19.0 to 20.0 +- * *web-app-peertube*: v8.3.0 to v8.3.1 +- * *web-app-seaweedfs*: 4.46 to 4.48 +- * *web-app-shopware*: OpenSearch 3.8.0 to 3.9.0 +- * *web-app-snipe-it*: v8.7.2-alpine to v8.8.0-alpine +- * *web-app-socialhome*: 2026.9.18 to 2026.10.2 +- * *web-app-xwiki*: 17.10.13 to 18.8.0 +- * *web-app-bluesky* (view): 1.132.0 to 1.134.0 +- * *web-app-discourse*: v2026.8.0 to v2026.9.0 +- * *web-svc-logout*: universal-logout v1.3.1 to v1.4.1 +- * *web-svc-xmpp*: 26.07 to 26.09 +- * *pkgmgr*: v1.16.1 to v2.0.2 +- * *backup-docker-to-local* (*baudolo*): 7.0.1 to 7.0.2 +- * *ruff* (dev): 0.16.7 to 0.16.9 +- * *brace-expansion* (dev): ^5.0.9 to ^5.0.12 + +- **Contributors** + +- * [Alejandro Roman Ibanez](https://github.com/AlejandroRomanIbanez): Stalwart as the default +- mail provider in compose and swarm mode, the Tor SMTP gateway, onion mail delivery for +- every mail-sending role, the Mailu import and the provider resolution +- * [Kevin Veen-Birkenbach](https://veen.world): PostgreSQL 18, release-aware CI cancellation +- and chunk gating, the release tag test, the Mailu deprecation and the untested deprecated +- stage, the agent sandbox policy, review and version maintenance + * Tue Sep 22 2026 Kevin Veen-Birkenbach - 14.2.0-1 - **For Users** diff --git a/plugins/lookup/container_extra_hosts.py b/plugins/lookup/container_extra_hosts.py index b63cc46afc..b42ed16996 100644 --- a/plugins/lookup/container_extra_hosts.py +++ b/plugins/lookup/container_extra_hosts.py @@ -86,6 +86,17 @@ class LookupModule(LookupBase): variables: dict[str, Any] | None = None, **kwargs: Any, ) -> list[str]: + """Render the pins this application needs to reach onion-hosted services. + + Kwargs: + application_id: role whose pins to resolve; defaults to the play var. + extra_hosts: additional `name:address` entries from the caller. + host_alias: also pin `host.docker.internal` to the gateway. + docker_flags: emit `--add-host=` flags for a `docker run` command + line instead of a compose `extra_hosts:` block. + + Returns a single-element list holding the rendered block, or [""]. + """ if terms: raise AnsibleError( "container_extra_hosts lookup takes no positional terms; pass " @@ -104,7 +115,7 @@ class LookupModule(LookupBase): "pass application_id= explicitly" ) - entries = self._sso_pins(application_id) + entries = self._sso_pins(application_id) + self._mail_pins(application_id) if _to_bool(self._render(kwargs.get("host_alias", False)), strict=False): entries.append( f"{DOCKER_INTERNAL_HOST}:{self._gateway_address(DOCKER_INTERNAL_HOST)}" @@ -116,6 +127,9 @@ class LookupModule(LookupBase): if not merged: return [""] + if _to_bool(self._render(kwargs.get("docker_flags", False)), strict=False): + return ["\n".join(f" - --add-host={entry}" for entry in merged)] + lines = ["extra_hosts:"] + [f' - "{entry}"' for entry in merged] return ["\n".join(lines)] @@ -143,6 +157,35 @@ class LookupModule(LookupBase): f"{provider_host}:{self._gateway_address(provider_host)}", ] + def _mail_pins(self, application_id: str) -> list[str]: + """The mail-provider pin, empty unless this is an onion node sending mail. + + Args: + application_id: the role whose ``services.email`` gate decides. + + The same unroutability as the SSO pin, one service down: an application + dials the relay by the provider's public name, which on an onion + deployment resolves only through Tor. ``sys-svc-mail-msmtp`` answers + that with a SOCKS proxy, but a client with no proxy setting at all -- + Ruby's ``Net::SMTP``, so GitLab queues the mail and reports success + while nothing leaves -- has no other route to the relay. + """ + if not bool( + self._lookup("config", application_id, "services.email.enabled", False) + ): + return [] + + mail_host = str( + (self._lookup("email", application_id) or {}).get("host") or "" + ).strip() + if not mail_host.endswith(".onion"): + return [] + + return [ + f"{DOCKER_INTERNAL_HOST}:{HOST_GATEWAY}", + f"{mail_host}:{self._gateway_address(mail_host)}", + ] + def _caller_entries(self, extra_hosts: Any) -> list[str]: """Normalise the ``extra_hosts=`` argument to a list of mappings. diff --git a/plugins/lookup/database.py b/plugins/lookup/database.py index 24feed1b98..3c7d7c9d93 100644 --- a/plugins/lookup/database.py +++ b/plugins/lookup/database.py @@ -85,6 +85,8 @@ class LookupModule(LookupBase): database_service = get_database_service_config(applications, consumer_id) enabled = bool(database_service.get("enabled", False)) shared = bool(database_service.get("shared", False)) + name_override = str(database_service.get("name") or "").strip() + managed = bool(enabled and not shared and not name_override) if not dbtype: resolved = { @@ -92,6 +94,7 @@ class LookupModule(LookupBase): "enabled": enabled, "shared": shared, "local": bool(enabled and not shared), + "managed": managed, "type": "", "name": consumer_entity, "instance": "", @@ -184,7 +187,7 @@ class LookupModule(LookupBase): default=default_version, ) - image = get( + default_image = get( applications, db_id, f"services.{dbtype}.image", @@ -193,6 +196,14 @@ class LookupModule(LookupBase): skip_missing_app=True, ) + image = get( + applications, + consumer_id, + f"services.{dbtype}.image", + strict=False, + default=default_image, + ) + env_dir = f"{path_instances}{entity_name(consumer_id)}/.env/" env = f"{env_dir}{dbtype}.env" realign_sql = f"{env_dir}{dbtype}-realign.sql" @@ -245,6 +256,7 @@ class LookupModule(LookupBase): "enabled": enabled, "shared": shared, "local": bool(enabled and not shared), + "managed": managed, "type": dbtype, "name": name, "instance": instance, diff --git a/plugins/lookup/email.py b/plugins/lookup/email.py index 09170e9b1e..b42b88d661 100644 --- a/plugins/lookup/email.py +++ b/plugins/lookup/email.py @@ -1,17 +1,25 @@ # nocheck: comments-valid explanatory WHY-comments in this SPOT lookup predate the stricter lint from __future__ import annotations +from pathlib import Path from typing import Any from ansible.errors import AnsibleError +from ansible.plugins.loader import lookup_loader from ansible.plugins.lookup import LookupBase from plugins.lookup.applications import LookupModule as ApplicationsLookup from plugins.lookup.domain import LookupModule as DomainLookup from plugins.lookup.users import LookupModule as UsersLookup +from utils.domains.primary_domain import get_domain +from utils.mail.provider import deployed_roles, resolve_active_provider +from utils.roles.entity.name import get_entity_name SYSTEM_EMAIL_PREFIX = "SYSTEM_EMAIL_" +# Default mail provider role; overridable per inventory via the MAIL_PROVIDER var. +DEFAULT_MAIL_PROVIDER = "web-app-stalwart" + # Declared in resolution order so each computed default sees already-resolved # predecessors (for example ``port`` depends on ``tls``). RESOLUTION_ORDER = ( @@ -131,13 +139,14 @@ class LookupModule(LookupBase): if short_key == "timeout": return "30" if short_key == "external": - # Cluster-wide, not per-host: in swarm mailu is manager-pinned so - # worker group_names lack it, yet every node relays through the - # central mailu (routing mesh). Gate on the group having a host so - # workers use the authenticated config, not a localhost root sender - # mailu rejects ("Sender address rejected: Domain not found"). + # Cluster-wide, not per-host: in swarm the mail provider is + # manager-pinned so worker group_names lack it, yet every node + # relays through the central provider (routing mesh). Gate on the + # group having a host so workers use the authenticated config, not + # a localhost root sender the provider rejects + # ("Sender address rejected: Domain not found"). groups = variables.get("groups") or {} - return bool(groups.get("web-app-mailu")) + return bool(groups.get(self._mail_provider(variables))) if short_key == "environment": external = _as_bool(resolved.get("external")) tls_enabled = _as_bool(variables.get("TLS_ENABLED")) @@ -152,14 +161,27 @@ class LookupModule(LookupBase): if not external: return False mail_host = str(resolved.get("host") or "").lower() - if mail_host.endswith(".onion"): + if mail_host.endswith(".onion") or self._provider_onion_primary(variables): return False return _as_bool(variables.get("TLS_ENABLED")) if short_key == "port": external = _as_bool(resolved.get("external")) + # SSO relay: port 25 is the only listener not requiring SMTP AUTH. + if self._provider_uses_sso_relay(variables): + return 25 if not external: return 25 - return 465 if _as_bool(resolved.get("tls")) else 587 + ports = self._provider_ports(variables) + if _as_bool(resolved.get("tls")): + return ports.get("smtps", 465) + # Plaintext submission is the onion/no-TLS path, but a provider is + # only reachable there if it binds one. Stalwart's default config + # binds the implicit-TLS client ports plus the MX, so it advertises + # no `submission`; fall back to the MX port rather than a listener + # nothing answers on (`auth` drops with it -- an MX takes local + # recipients without SMTP AUTH, the same property the SSO relay + # above relies on). + return ports.get("submission") or ports.get("smtp", 25) if short_key == "host": env = resolved.get("environment") # `environment` folds TLS_ENABLED into its "external" base, so a @@ -173,18 +195,32 @@ class LookupModule(LookupBase): "localhost_container", ): return "localhost" - return self._lookup_mailu_domain(variables) + return self._lookup_mail_provider_domain(variables) if short_key == "auth": env = resolved.get("environment") if env in ("external_container", "localhost"): return False + if self._provider_uses_sso_relay(variables): + return False + # The MX listener offers no SMTP AUTH: when `port` fell back to it + # because the provider binds no plaintext submission, announcing + # AUTH aborts msmtp with EX_UNAVAILABLE before it ever sends. + ports = self._provider_ports(variables) + if not ports.get("submission") and resolved.get("port") == ports.get( + "smtp", 25 + ): + return False return _as_bool(resolved.get("external")) if short_key == "auth_mechanism": if not _as_bool(resolved.get("auth")): return "off" return "on" if _as_bool(resolved.get("tls")) else "plain" if short_key == "start_tls": - return False + if str(resolved.get("host") or "").lower().endswith( + ".onion" + ) or self._provider_onion_primary(variables): + return False + return self._provider_uses_sso_relay(variables) if short_key == "smtp": return True if short_key == "from": @@ -204,15 +240,132 @@ class LookupModule(LookupBase): if isinstance(no_reply, dict): tokens = no_reply.get("tokens") or {} if isinstance(tokens, dict): - return tokens.get("web-app-mailu", "") or "" + return tokens.get(self._mail_provider(variables), "") or "" return "" raise AnsibleError(f"email: unknown key {short_key!r}") - def _lookup_mailu_domain(self, variables: dict[str, Any]) -> Any: + def _mail_provider(self, variables: dict[str, Any]) -> str: + value = variables.get("MAIL_PROVIDER") + configured = str(value).strip() if value else DEFAULT_MAIL_PROVIDER + return resolve_active_provider( + configured, + deployed_roles(variables.get("groups")), + Path.cwd() / "roles", + ) + + def _provider_services(self, variables: dict[str, Any]) -> dict[str, Any]: + """The active provider's ``services`` block, or ``{}`` when it cannot be + resolved. Both the relay probe and the port probe read the provider's + own self-declaration, so the applications lookup is done once here.""" + apps = ApplicationsLookup() + apps._templar = getattr(self, "_templar", None) + forwarded = { + k: v for k, v in getattr(self, "_kwargs", {}).items() if k == "roles_dir" + } + try: + entry = apps.run( + [self._mail_provider(variables), {}], + variables=variables, + **forwarded, + )[0] + except AnsibleError: + return {} + if not isinstance(entry, dict): + return {} + services = entry.get("services") or {} + return services if isinstance(services, dict) else {} + + def _provider_ports(self, variables: dict[str, Any]) -> dict[str, int]: + """The SMTP ports the active provider publishes, keyed by protocol. + + Read from the provider's own ``services..ports.public`` rather + than assumed, so a provider that binds no plaintext ``submission`` + (Stalwart) resolves differently from one that does (Mailu) without + either being named here. + """ + services = self._provider_services(variables) + entity = get_entity_name(self._mail_provider(variables)) + entry = services.get(entity) or {} + if not isinstance(entry, dict): + return {} + ports = (entry.get("ports") or {}).get("public") or {} + if not isinstance(ports, dict): + return {} + out: dict[str, int] = {} + for name, value in ports.items(): + try: + out[str(name)] = int(value) + except (TypeError, ValueError): + continue + return out + + def _provider_uses_sso_relay(self, variables: dict[str, Any]) -> bool: + """True when the active provider self-declares SSO relay (via + ``services.sso.oidc.submission_via_relay``) and Keycloak is deployed.""" + group_names = variables.get("group_names") or [] + if "web-app-keycloak" not in group_names: + return False + services = self._provider_services(variables) + sso = services.get("sso") or {} if isinstance(services, dict) else {} + oidc = sso.get("oidc") or {} if isinstance(sso, dict) else {} + if not isinstance(oidc, dict): + return False + # A variant/inventory can pin the provider's sso.enabled to a literal + # false while submission_via_relay stays true in the role defaults — + # then the provider keeps password auth and never widens allowRelaying, + # so relay mode must be off. Untemplated Jinja (the role default gates + # on group_names) falls through to the Keycloak check above. + enabled = sso.get("enabled") if isinstance(sso, dict) else None + if enabled is not None and "{{" not in str(enabled) and not _as_bool(enabled): + return False + return _as_bool(oidc.get("submission_via_relay")) + + def _provider_onion_primary(self, variables: dict[str, Any]) -> bool: + """True when the provider's own host serves it onion-first. + + Resolved from the provider host's ``group_names`` rather than this + host's: in swarm only the manager is in ``svc-net-tor``, and the + provider's ``tor.enabled`` renders against ``group_names``, so a + worker would otherwise see the clearnet name the provider never + serves a certificate for. + """ + groups = variables.get("groups") or {} + provider = self._mail_provider(variables) + tor_hosts = set(groups.get("svc-net-tor") or []) + provider_hosts = [h for h in groups.get(provider) or [] if h in tor_hosts] + if not provider_hosts: + return False + own_groups = list(variables.get("group_names") or []) + fallback = own_groups + [ + g for g in ("svc-net-tor", provider) if g not in own_groups + ] + try: + provider_groups = variables["hostvars"][provider_hosts[0]]["group_names"] + except (KeyError, TypeError): + provider_groups = fallback + provider_view = dict(variables) + provider_view["group_names"] = list(provider_groups or fallback) + forwarded = { + k: v for k, v in getattr(self, "_kwargs", {}).items() if k == "roles_dir" + } + try: + domains = lookup_loader.get( + "domains", + loader=getattr(self, "_loader", None), + templar=getattr(self, "_templar", None), + ).run([], variables=provider_view, **forwarded)[0] + domain = get_domain(domains, provider) + except Exception: + return False + return str(domain or "").lower().endswith(".onion") + + def _lookup_mail_provider_domain(self, variables: dict[str, Any]) -> Any: domain_lookup = DomainLookup() domain_lookup._templar = getattr(self, "_templar", None) try: - return domain_lookup.run(["web-app-mailu"], variables=variables)[0] + return domain_lookup.run( + [self._mail_provider(variables)], variables=variables + )[0] except Exception: return "localhost" diff --git a/plugins/lookup/mail_provider.py b/plugins/lookup/mail_provider.py new file mode 100644 index 0000000000..c1cecf76eb --- /dev/null +++ b/plugins/lookup/mail_provider.py @@ -0,0 +1,37 @@ +"""Resolve the role acting as mail provider for this deploy. + +Usage: + {{ lookup('mail_provider') }} + +See :mod:`utils.mail.provider` for why this is not simply ``MAIL_PROVIDER``. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ansible.plugins.lookup import LookupBase + +from utils.mail.provider import deployed_roles, resolve_active_provider + +DEFAULT_MAIL_PROVIDER = "web-app-stalwart" + + +class LookupModule(LookupBase): + def run( + self, + terms: list[Any] | None = None, + variables: dict[str, Any] | None = None, + **kwargs: Any, + ) -> list[str]: + vars_ = variables or getattr(self._templar, "available_variables", {}) or {} + configured = ( + str(vars_.get("MAIL_PROVIDER") or "").strip() or DEFAULT_MAIL_PROVIDER + ) + roles_dir = Path(kwargs.get("roles_dir") or Path.cwd() / "roles") + return [ + resolve_active_provider( + configured, deployed_roles(vars_.get("groups")), roles_dir + ) + ] diff --git a/plugins/lookup/postgres_libraries.py b/plugins/lookup/postgres_libraries.py index a52ae352dc..0261b77a2d 100644 --- a/plugins/lookup/postgres_libraries.py +++ b/plugins/lookup/postgres_libraries.py @@ -8,7 +8,6 @@ Takes a list of extension names and returns the subset that needs a custom library install on top of the official postgres base image (the contrib extensions bloom, pg_trgm and unaccent are already in the base — those are filtered out). Each returned entry is a dict with: - - extension: the extension name (matches the SQL identifier) - name: the library/source name (used in the Dockerfile context) - apt_package: the PGDG package suffix, installed as diff --git a/plugins/lookup/smtp_host.py b/plugins/lookup/smtp_host.py index b3ed83d117..31eec66989 100644 --- a/plugins/lookup/smtp_host.py +++ b/plugins/lookup/smtp_host.py @@ -6,6 +6,8 @@ from ansible.errors import AnsibleError from ansible.module_utils.parsing.convert_bool import boolean as _to_bool from ansible.plugins.lookup import LookupBase +from utils.networks.lookup_context import resolve_var + LOOPBACK = "127.0.0.1" CONTAINER_HOST_ALIAS = "host.docker.internal" NODE_GROUP = "svc-swarm-node" @@ -78,4 +80,9 @@ class LookupModule(LookupBase): raise AnsibleError("smtp_host: the term must be the email mapping.") variables = variables or getattr(self._templar, "available_variables", {}) or {} in_container = _to_bool(kwargs.get("in_container", False), strict=False) + templar = getattr(self, "_templar", None) + variables = dict(variables) + for key in ("DOCKER_IN_CONTAINER", "DEPLOYMENT_MODE"): + if key in variables: + variables[key] = resolve_var(templar, variables[key]) return [resolve_smtp_host(variables, email, in_container=in_container)] diff --git a/plugins/lookup/tor_ports.py b/plugins/lookup/tor_ports.py index fea32575f8..139443a59a 100644 --- a/plugins/lookup/tor_ports.py +++ b/plugins/lookup/tor_ports.py @@ -45,6 +45,43 @@ def _collect_local_tcp_ports(port_categories: Any, into: set[int]) -> None: into.add(value) +def _provided_capabilities( + applications: dict[str, Any], deployed: set[str] +) -> dict[str, set[str]]: + """Capability key -> the deployed roles whose service ``provides:`` it. + + Args: + applications: the variant-merged applications view. + deployed: role ids present in this deploy. + """ + provided: dict[str, set[str]] = {} + for app_id, cfg in applications.items(): + if app_id not in deployed or not isinstance(cfg, dict): + continue + services = cfg.get("services") + if not isinstance(services, dict): + continue + for entity in services.values(): + if not isinstance(entity, dict): + continue + key = entity.get("provides") + if isinstance(key, str) and key: + provided.setdefault(key, set()).add(app_id) + return provided + + +def _is_inactive_coverer( + entity: dict[str, Any], app_id: str, provided: dict[str, set[str]] +) -> bool: + """Whether this service only covers a capability another deployed role provides.""" + covers = entity.get("covers") + if not isinstance(covers, (list, tuple)): + return False + return any( + isinstance(key, str) and (provided.get(key, set()) - {app_id}) for key in covers + ) + + def collect_onion_ports( applications: dict[str, Any], deployed_roles: list[str], @@ -64,11 +101,19 @@ def collect_onion_ports( declared next to ``network.public: false``, and Mailu's implicit-TLS ports are dropped from the publish list whenever TLS is off, which an onion deployment always is. + + A service that only ``covers:`` a capability another deployed role + ``provides:`` is skipped for the same reason: it is a legacy co-tenant, the + provider owns the host ports, and forwarding the co-tenant's onion ports + advertises endpoints nothing answers on. The Mailu-to-Stalwart migration + round deploys both, and the ports Mailu alone would serve (110, 143, 587) + are dead there. """ ports: set[int] = set() deployed = set(deployed_roles) if not isinstance(applications, dict): return [] + provided = _provided_capabilities(applications, deployed) for app_id, cfg in applications.items(): if app_id not in deployed: continue @@ -78,6 +123,8 @@ def collect_onion_ports( for entity in services.values(): if not isinstance(entity, dict): continue + if _is_inactive_coverer(entity, app_id, provided): + continue declared = entity.get("ports") if not isinstance(declared, dict): continue diff --git a/pyproject.toml b/pyproject.toml index 899f422582..b8251a24f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "infinito-nexus" -version = "14.2.0" +version = "15.0.0" description = "Infinito.Nexus" readme = "README.md" requires-python = ">=3.11" diff --git a/roles/pkgmgr/meta/services.yml b/roles/pkgmgr/meta/services.yml index 86f6109211..b2c3923eba 100644 --- a/roles/pkgmgr/meta/services.yml +++ b/roles/pkgmgr/meta/services.yml @@ -3,4 +3,4 @@ pkgmgr: lifecycle: beta repository: https://github.com/kevinveenbirkenbach/package-manager.git - ref: v2.0.1 + ref: v2.0.2 diff --git a/roles/svc-bkp-volume-2-local/files/test/restore_cycle.sh b/roles/svc-bkp-volume-2-local/files/test/restore_cycle.sh index 92e484dae8..e1be83b65f 100755 --- a/roles/svc-bkp-volume-2-local/files/test/restore_cycle.sh +++ b/roles/svc-bkp-volume-2-local/files/test/restore_cycle.sh @@ -46,7 +46,7 @@ if [[ ! -d "${BKP_TEST_REPO_ROOT}/cli/administration/recover" ]]; then fi recover() { - PYTHONPATH="${BKP_TEST_REPO_ROOT}" python3 -m cli.administration.recover \ + PYTHONPATH="${BKP_TEST_REPO_ROOT}" "${BKP_TEST_PYTHON}" -m cli.administration.recover \ "$1" "$2" localhost --no-safety-backup } diff --git a/roles/svc-net-tor-smtp/README.md b/roles/svc-net-tor-smtp/README.md new file mode 100644 index 0000000000..c4410489e1 --- /dev/null +++ b/roles/svc-net-tor-smtp/README.md @@ -0,0 +1,40 @@ +# Tor SMTP Gateway (svc-net-tor-smtp) + +## Description + +A small SMTP relay that carries `.onion` recipients over Tor. The active mail +provider (Stalwart) cannot open a SOCKS connection of its own, and a `.onion` +address cannot be reached any other way, so the provider hands every `.onion` +recipient to this gateway as an ordinary SMTP relay target and the gateway dials +`:25` through Tor's SOCKS5 listener. + +## Overview + +``` +Stalwart ──(relay route: rcpt ends_with .onion)──▶ tor-smtp:1525 +tor-smtp ──(SOCKS5, remote DNS)──▶ Tor ──▶ .onion:25 +``` + +Clearnet mail is untouched — it stays on the provider's normal MX delivery. The +gateway only ever accepts `.onion` recipients; it is not an open relay. + +## Features + +- **Onion-only relay:** accepts a recipient only when its domain ends with + `.onion`; every clearnet recipient is refused with `550`. +- **Remote resolution:** dials through Tor's SOCKS5 with `proxy_rdns`, so the + `.onion` name is resolved by Tor and never looked up locally. +- **One hop per domain:** recipients sharing a `.onion` domain are delivered in a + single SMTP transaction. +- **Clearnet untouched:** normal SMTP reputation, DKIM and SPF delivery never + pass through Tor. + +## Design Decisions + +- **A dedicated role, not Stalwart config.** Stalwart has no outbound SOCKS + support ([stalwartlabs/stalwart#644](https://github.com/stalwartlabs/stalwart/issues/644)), + so the Tor hop lives in a separate service that only the `.onion` route uses. +- **`proxy_rdns` is on.** A `.onion` has no DNS entry, so Tor must resolve it — + the name is never resolved locally. +- **Co-located with `svc-net-tor`.** It dials that role's SOCKS listener, so it + is manager-pinned like the Tor daemon. diff --git a/roles/svc-net-tor-smtp/files/Dockerfile b/roles/svc-net-tor-smtp/files/Dockerfile new file mode 100644 index 0000000000..c6f43f7b3a --- /dev/null +++ b/roles/svc-net-tor-smtp/files/Dockerfile @@ -0,0 +1,14 @@ +# Tor SMTP gateway: receives SMTP from the active mail provider and relays each +# .onion recipient through Tor's SOCKS5 listener. aiosmtpd is the SMTP server, +# PySocks the SOCKS client — both pinned so the image is reproducible. +ARG PY_BASE +FROM ${PY_BASE} + +RUN pip install --no-cache-dir "aiosmtpd==1.4.6" "PySocks==1.7.1" + +COPY tor_smtp_gateway.py /usr/local/bin/tor_smtp_gateway.py + +RUN adduser --system --no-create-home --group gateway +USER gateway + +ENTRYPOINT ["python", "/usr/local/bin/tor_smtp_gateway.py"] diff --git a/roles/svc-net-tor-smtp/files/python/tor_smtp_gateway.py b/roles/svc-net-tor-smtp/files/python/tor_smtp_gateway.py new file mode 100755 index 0000000000..14f22ed970 --- /dev/null +++ b/roles/svc-net-tor-smtp/files/python/tor_smtp_gateway.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""SMTP-to-Tor gateway: accept a message, deliver it to the recipient's ``.onion``. + +The active mail provider (Stalwart) cannot open a SOCKS connection of its own +(stalwartlabs/stalwart#644), and a ``.onion`` recipient can be reached no other +way — it has no DNS entry, so MX resolution and a direct TCP connect both fail. +So the provider's outbound routing hands every ``.onion`` recipient to this +gateway as a plain SMTP relay target, and the gateway dials +``:25`` through Tor's SOCKS5 listener. ``proxy_rdns`` is on: +the hostname is resolved by Tor, never locally, which is the whole point. + +Only ``.onion`` recipients are accepted — the gateway is not an open relay and +never touches clearnet delivery, which stays on the provider's normal MX path. + +Environment (compose supplies the first four from the role config; the last two +carry operational defaults): + TOR_SMTP_LISTEN_PORT bind port (required) + TOR_SOCKS_HOST Tor SOCKS host (required) + TOR_SOCKS_PORT Tor SOCKS port (required) + TOR_SMTP_HELO EHLO name announced to the peer (required — the node's + mail identity; never the bind address, which strict + receivers reject) + TOR_SMTP_LISTEN_HOST bind address (default 0.0.0.0) + TOR_SMTP_TIMEOUT per-hop seconds (default 120) +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import signal +import smtplib + +import socks +from aiosmtpd.controller import Controller + +log = logging.getLogger("tor-smtp-gateway") + +ONION_SMTP_PORT = 25 + +LISTEN_HOST = os.environ.get("TOR_SMTP_LISTEN_HOST", "0.0.0.0") # noqa: S104 - relay bind; reachability is gated by the compose network, not this bind +LISTEN_PORT = int(os.environ["TOR_SMTP_LISTEN_PORT"]) +SOCKS_HOST = os.environ["TOR_SOCKS_HOST"] +SOCKS_PORT = int(os.environ["TOR_SOCKS_PORT"]) +HELO_NAME = os.environ["TOR_SMTP_HELO"] +TIMEOUT = int(os.environ.get("TOR_SMTP_TIMEOUT", "120")) + + +class _SocksSMTP(smtplib.SMTP): + """An ``smtplib.SMTP`` whose socket is a Tor SOCKS5 tunnel to a ``.onion``.""" + + def __init__(self, onion: str, port: int, timeout: int) -> None: + self._onion = onion + self._onion_port = port + super().__init__(local_hostname=HELO_NAME, timeout=timeout) + + def _get_socket(self, host, port, timeout): + return socks.create_connection( + (self._onion, self._onion_port), + timeout=timeout, + proxy_type=socks.SOCKS5, + proxy_addr=SOCKS_HOST, + proxy_port=SOCKS_PORT, + proxy_rdns=True, + ) + + +def _deliver(onion: str, mail_from: str, rcpts: list[str], content: bytes) -> None: + smtp = _SocksSMTP(onion, ONION_SMTP_PORT, TIMEOUT) + try: + smtp.connect(onion, ONION_SMTP_PORT) + smtp.ehlo_or_helo_if_needed() + smtp.sendmail(mail_from, rcpts, content) + finally: + try: + smtp.quit() + except Exception: + smtp.close() + + +def _group_by_domain(rcpts: list[str]) -> dict[str, list[str]]: + grouped: dict[str, list[str]] = {} + for rcpt in rcpts: + domain = rcpt.rsplit("@", 1)[-1].lower() + grouped.setdefault(domain, []).append(rcpt) + return grouped + + +class OnionRelay: + """aiosmtpd handler that relays each recipient's message over Tor.""" + + async def handle_RCPT(self, server, session, envelope, address, rcpt_options): # noqa: N802 - aiosmtpd hook name + if not address.rsplit("@", 1)[-1].lower().endswith(".onion"): + return "550 5.7.1 tor-smtp-gateway relays .onion recipients only" + envelope.rcpt_tos.append(address) + return "250 OK" + + async def handle_DATA(self, server, session, envelope): # noqa: N802 - aiosmtpd hook name + loop = asyncio.get_running_loop() + for domain, rcpts in _group_by_domain(envelope.rcpt_tos).items(): + try: + await loop.run_in_executor( + None, _deliver, domain, envelope.mail_from, rcpts, envelope.content + ) + except Exception as exc: + log.warning("onion relay to %s failed: %s", domain, exc) + return f"451 4.4.1 onion relay to {domain} failed" + return "250 2.0.0 accepted for onion delivery" + + +def main() -> None: + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s" + ) + controller = Controller(OnionRelay(), hostname=LISTEN_HOST, port=LISTEN_PORT) + controller.start() + log.info( + "tor-smtp-gateway listening on %s:%s → SOCKS %s:%s", + LISTEN_HOST, + LISTEN_PORT, + SOCKS_HOST, + SOCKS_PORT, + ) + stop = asyncio.Event() + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, stop.set) + try: + loop.run_until_complete(stop.wait()) + finally: + controller.stop() + loop.close() + + +if __name__ == "__main__": + main() diff --git a/roles/svc-net-tor-smtp/meta/info.yml b/roles/svc-net-tor-smtp/meta/info.yml new file mode 100644 index 0000000000..b5baf5213c --- /dev/null +++ b/roles/svc-net-tor-smtp/meta/info.yml @@ -0,0 +1,4 @@ +--- +# nocheck: info-media in-repo gateway, no upstream project to link or demo +logo: + class: fa-solid fa-user-secret diff --git a/roles/svc-net-tor-smtp/meta/main.yml b/roles/svc-net-tor-smtp/meta/main.yml new file mode 100644 index 0000000000..e684ee53df --- /dev/null +++ b/roles/svc-net-tor-smtp/meta/main.yml @@ -0,0 +1,32 @@ +--- +# nocheck: info-media no official demo video +galaxy_info: + author: Alejandro Roman Ibanez + description: SMTP gateway that relays .onion recipients through Tor's SOCKS listener + license: Infinito.Nexus Community License (Non-Commercial) + galaxy_tags: + - tor + - onion + - smtp + - mail + - networking + - self-hosted + issue_tracker_url: https://s.infinito.nexus/issues + min_ansible_version: "2.9" + platforms: + - name: ArchLinux + versions: + - all + - name: Debian + versions: + - all + - name: EL + versions: + - all + - name: Fedora + versions: + - all + - name: Ubuntu + versions: + - all +dependencies: [] diff --git a/roles/svc-net-tor-smtp/meta/networks.yml b/roles/svc-net-tor-smtp/meta/networks.yml new file mode 100644 index 0000000000..7185e59673 --- /dev/null +++ b/roles/svc-net-tor-smtp/meta/networks.yml @@ -0,0 +1,10 @@ +--- +local: + subnet: 192.168.109.0/24 +overlay: + modes: [compose, swarm] + topology: shared_net + aliases: [tor-smtp] + consumer: + kind: services_flags + key: tor-smtp diff --git a/roles/svc-net-tor-smtp/meta/services.yml b/roles/svc-net-tor-smtp/meta/services.yml new file mode 100644 index 0000000000..c70fd26cd7 --- /dev/null +++ b/roles/svc-net-tor-smtp/meta/services.yml @@ -0,0 +1,30 @@ +--- +# nocheck: dockerfile-custom +# nocheck: playwright-service-flag — SMTP egress daemon, no browser surface +tor-smtp: # nocheck: category-modes + primary: true + bond: 1 + enabled: true + shared: true + custom: true + modes: + compose: + enabled: true + swarm: + enabled: true + # nocheck: default-placement-manager dials svc-net-tor's host-loopback SOCKS listener, which is manager-pinned; the gateway must share that node + placement: manager + image: python + version: 3.14-slim + name: tor-smtp + ports: + internal: + smtp: 1525 + healthcheck: + test: ["CMD", "python", "-c", "import os,socket; socket.create_connection(('127.0.0.1', int(os.environ['TOR_SMTP_LISTEN_PORT'])), 3).close()"] + min_storage: 100MB + cpus: "0.25" + mem_reservation: 64m + mem_limit: 128m + pids_limit: 64 + lifecycle: alpha diff --git a/roles/svc-net-tor-smtp/tasks/00_core.yml b/roles/svc-net-tor-smtp/tasks/00_core.yml new file mode 100644 index 0000000000..fcc5dfab96 --- /dev/null +++ b/roles/svc-net-tor-smtp/tasks/00_core.yml @@ -0,0 +1,25 @@ +--- +- include_tasks: utils/once/flag.yml + +- name: "📂 Include sys-svc-compose (build deferred to stage the daemon first)" + include_role: + name: sys-svc-compose + vars: + compose_handlers_flush: false + compose_repository_pull: false + application_id: "svc-net-tor-smtp" + +- name: "📁 Create container network" + include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/network/create.yml') }}" + vars: + network_role_id: "{{ application_id }}" + +- name: "🎬 Stage the gateway daemon into the build context" + stack_host_copy: + src: "python/tor_smtp_gateway.py" + dest: "{{ [ lookup('container', application_id, 'directories.instance'), 'tor_smtp_gateway.py' ] | path_join }}" + mode: "0644" + when: IS_STACK_HOST | bool + +- name: "🚿 Flush compose handlers (build + up with the daemon present)" + meta: flush_handlers diff --git a/roles/svc-net-tor-smtp/tasks/main.yml b/roles/svc-net-tor-smtp/tasks/main.yml new file mode 100644 index 0000000000..cbe6fcbbef --- /dev/null +++ b/roles/svc-net-tor-smtp/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- include_tasks: 00_core.yml + when: run_once_svc_net_tor_smtp is not defined + vars: + application_id: svc-net-tor-smtp + +- include_tasks: utils/load/handlers.yml + vars: + handler_role_name: "sys-svc-compose" diff --git a/roles/svc-net-tor-smtp/templates/compose.yml.j2 b/roles/svc-net-tor-smtp/templates/compose.yml.j2 new file mode 100644 index 0000000000..65b22dafaf --- /dev/null +++ b/roles/svc-net-tor-smtp/templates/compose.yml.j2 @@ -0,0 +1,15 @@ +{% include 'roles/sys-svc-compose/templates/base.yml.j2' %} + {% set service_name = 'tor-smtp' %} + {{ service_name }}: + {{ lookup('template', 'roles/sys-svc-container/templates/build.yml.j2') | indent(4) }} + args: + PY_BASE: "{{ TOR_SMTP_IMAGE }}:{{ TOR_SMTP_VERSION }}" + {{ lookup('container_image', application_id, service_name) }} + {{ lookup('compose_only', 'container_name', TOR_SMTP_CONTAINER) }} + {% include 'roles/sys-svc-container/templates/base.yml.j2' %} + {{ lookup('container_extra_hosts') | indent(4) }} + {{ lookup('container_healthcheck', service_name) | indent(4) }} + {{ lookup('container_networks') | indent(4) }} + +{{ lookup('compose_volumes') }} +{{ lookup('compose_networks') }} diff --git a/roles/svc-net-tor-smtp/templates/env.j2 b/roles/svc-net-tor-smtp/templates/env.j2 new file mode 100644 index 0000000000..e0f491e2be --- /dev/null +++ b/roles/svc-net-tor-smtp/templates/env.j2 @@ -0,0 +1,6 @@ +{% include 'roles/sys-svc-container/templates/env.j2' %} + +TOR_SMTP_LISTEN_PORT={{ TOR_SMTP_LISTEN_PORT }} +TOR_SOCKS_HOST={{ TOR_SMTP_SOCKS_HOST }} +TOR_SOCKS_PORT={{ TOR_SMTP_SOCKS_PORT }} +TOR_SMTP_HELO={{ TOR_SMTP_HELO }} diff --git a/roles/svc-net-tor-smtp/vars/main.yml b/roles/svc-net-tor-smtp/vars/main.yml new file mode 100644 index 0000000000..17723c1fe5 --- /dev/null +++ b/roles/svc-net-tor-smtp/vars/main.yml @@ -0,0 +1,13 @@ +--- +application_id: "svc-net-tor-smtp" + +TOR_SMTP_IMAGE: "{{ lookup('config', application_id, 'services.tor-smtp.image') }}" +TOR_SMTP_VERSION: "{{ lookup('config', application_id, 'services.tor-smtp.version') }}" +TOR_SMTP_CONTAINER: "{{ lookup('config', application_id, 'services.tor-smtp.name') }}" +TOR_SMTP_LISTEN_PORT: "{{ lookup('config', application_id, 'services.tor-smtp.ports.internal.smtp') }}" + +TOR_SMTP_SOCKS: "{{ lookup('tor_socks_proxy') | regex_replace('^socks5h://', '') }}" +TOR_SMTP_SOCKS_HOST: "{{ TOR_SMTP_SOCKS.split(':')[0] }}" +TOR_SMTP_SOCKS_PORT: "{{ TOR_SMTP_SOCKS.split(':')[1] }}" + +TOR_SMTP_HELO: "{{ DOMAIN_PRIMARY }}" diff --git a/roles/svc-storage-nfs-server/defaults/main.yml b/roles/svc-storage-nfs-server/defaults/main.yml new file mode 100644 index 0000000000..c0aff0a7be --- /dev/null +++ b/roles/svc-storage-nfs-server/defaults/main.yml @@ -0,0 +1,3 @@ +--- +# Param: NFSv4 minor versions ganesha offers to clients (comma-separated). +SVC_STORAGE_NFS_SERVER_MINOR_VERSIONS: "0, 1, 2" diff --git a/roles/svc-storage-nfs-server/templates/ganesha.conf.j2 b/roles/svc-storage-nfs-server/templates/ganesha.conf.j2 index 10b22ea5fe..44b076258f 100644 --- a/roles/svc-storage-nfs-server/templates/ganesha.conf.j2 +++ b/roles/svc-storage-nfs-server/templates/ganesha.conf.j2 @@ -18,7 +18,7 @@ NFSV4 { Allow_Numeric_Owners = true; Only_Numeric_Owners = true; Graceless = true; - Minor_Versions = 0, 1, 2; + Minor_Versions = {{ SVC_STORAGE_NFS_SERVER_MINOR_VERSIONS }}; } EXPORT { diff --git a/roles/sys-ca-selfsigned/files/shell/with-ca-trust.sh b/roles/sys-ca-selfsigned/files/shell/with-ca-trust.sh index 07b979e48b..adb78b97ac 100644 --- a/roles/sys-ca-selfsigned/files/shell/with-ca-trust.sh +++ b/roles/sys-ca-selfsigned/files/shell/with-ca-trust.sh @@ -6,6 +6,8 @@ set -eu VERBOSE="${VERBOSE:-1}" +SYS_CA_BUNDLE_CANDIDATES="/etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/cert.pem" + log() { if [ "$VERBOSE" = "1" ]; then echo "[with-ca-trust] $*" >&2 @@ -40,6 +42,18 @@ if [ -n "${CA_TRUST_BUNDLE:-}" ] && [ -r "${CA_TRUST_BUNDLE}" ]; then _ca_trust_verify_file="$CA_TRUST_BUNDLE" else _ca_trust_verify_file="$CA_TRUST_CERT" + # shellcheck disable=SC2086 # intentional word-splitting; paths contain no spaces + for sys_bundle in $SYS_CA_BUNDLE_CANDIDATES; do + if [ -r "$sys_bundle" ] && + _ca_trust_sys_combined="$(mktemp -t with-ca-trust-combined.XXXXXX 2>/dev/null)" && + cat "$sys_bundle" "$CA_TRUST_CERT" > "$_ca_trust_sys_combined" 2>/dev/null && + chmod 0644 "$_ca_trust_sys_combined" 2>/dev/null + then + _ca_trust_verify_file="$_ca_trust_sys_combined" + log "Combined system CA bundle ($sys_bundle) with ${name} -> $_ca_trust_sys_combined" + break + fi + done fi export SSL_CERT_FILE="$_ca_trust_verify_file" @@ -49,7 +63,8 @@ export NODE_EXTRA_CA_CERTS="$CA_TRUST_CERT" if [ -n "${CA_TRUST_CERT_EXTRA:-}" ] && [ -r "${CA_TRUST_CERT_EXTRA}" ]; then if combined="$(mktemp -t ca-trust-combined.XXXXXX 2>/dev/null)" && - cat "$_ca_trust_verify_file" "$CA_TRUST_CERT_EXTRA" > "$combined" 2>/dev/null + cat "$_ca_trust_verify_file" "$CA_TRUST_CERT_EXTRA" > "$combined" 2>/dev/null && + chmod 0644 "$combined" 2>/dev/null then export SSL_CERT_FILE="$combined" export REQUESTS_CA_BUNDLE="$combined" diff --git a/roles/sys-ctl-mtn-cert-deploy/files/shell/script.sh b/roles/sys-ctl-mtn-cert-deploy/files/shell/script.sh index 6815eb3e99..daa63f9d54 100644 --- a/roles/sys-ctl-mtn-cert-deploy/files/shell/script.sh +++ b/roles/sys-ctl-mtn-cert-deploy/files/shell/script.sh @@ -1,7 +1,13 @@ #!/bin/sh set -eu +# Deploy renewed TLS material into a compose project's certs volume and +# restart the services that consume it. +# # Usage: script.sh +# +# Requires $compose_cmd: the compose wrapper base command, quoted as needed, +# e.g. compose_cmd="compose --chdir /opt/compose/mailu --project mailu". if [ "$#" -ne 2 ]; then echo "Usage: $0 " >&2 exit 1 @@ -56,18 +62,26 @@ restart_services="" for service in $services; do echo "Checking service: $service" - if sh -c "$compose_cmd -- exec -T \"$service\" which nginx" > /dev/null 2>&1; then - echo "Nginx found in service: $service" + container_id="$(sh -c "$compose_cmd -- ps -q \"$service\"" | head -n1)" + if [ -z "$container_id" ]; then + echo "No running container for service: $service, skipping." + continue + fi + + if docker inspect --type container "$container_id" \ + --format '{{range .Mounts}}{{println .Source}}{{end}}' \ + | grep -qx "$docker_compose_cert_directory"; then + echo "Certificate mount found in service: $service" restart_services="$restart_services $service" else - echo "Nginx not found in service: $service, skipping." + echo "No certificate mount in service: $service, skipping." fi done if [ -n "$(echo "$restart_services" | tr -d ' ')" ]; then - echo "Restarting Nginx services to apply new certificates:${restart_services}" + echo "Restarting certificate-consuming services to apply new certificates:${restart_services}" # shellcheck disable=SC2086 sh -c "$compose_cmd -- restart $restart_services" || exit 1 else - echo "No Nginx instances found in any service. Nothing to restart." + echo "No certificate-consuming services found. Nothing to restart." fi diff --git a/roles/sys-ctl-mtn-cert-deploy/files/shell/verify.sh b/roles/sys-ctl-mtn-cert-deploy/files/shell/verify.sh new file mode 100755 index 0000000000..3e0dbe8b10 --- /dev/null +++ b/roles/sys-ctl-mtn-cert-deploy/files/shell/verify.sh @@ -0,0 +1,116 @@ +#!/bin/sh +set -eu + +# Validates deployed TLS certificates. +# +# Local mode (argument): checks the cert material script.sh deployed into a +# compose instance directory — cert.pem/key.pem exist, parse, match each +# other, and are not about to expire. +# +# Remote mode (environment): checks live TLS endpoints, e.g. production +# servers, so a misconfigured host surfaces in CI instead of in an outage. +# +# Usage: +# verify.sh [] +# +# Environment: +# CERT_TEST_REMOTE_ENDPOINTS Space-separated "host[:port]" list (port +# defaults to 443). Intended to be fed from a +# CI/GitHub variable. +# CERT_TEST_MIN_DAYS Minimum remaining validity in days (default 14). +# +# At least one mode must be selected. Exit code is non-zero on any failure. + +min_days="${CERT_TEST_MIN_DAYS:-14}" +min_seconds=$((min_days * 86400)) +failures=0 + +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +check_local() { + instance_dir="$1" + cert_dir="${instance_dir%/}/volumes/certs" + cert="${cert_dir}/cert.pem" + key="${cert_dir}/key.pem" + + echo "Checking deployed certificate in: $cert_dir" + + if [ ! -r "$cert" ] || [ ! -r "$key" ]; then + fail "cert.pem/key.pem missing or unreadable in $cert_dir" + return + fi + + if ! openssl x509 -in "$cert" -noout 2>/dev/null; then + fail "$cert does not parse as an X.509 certificate" + return + fi + + cert_pub="$(openssl x509 -in "$cert" -pubkey -noout 2>/dev/null)" + key_pub="$(openssl pkey -in "$key" -pubout 2>/dev/null)" + if [ -z "$key_pub" ] || [ "$cert_pub" != "$key_pub" ]; then + fail "key.pem does not match cert.pem in $cert_dir" + fi + + if ! openssl x509 -in "$cert" -noout -checkend "$min_seconds" >/dev/null; then + fail "$cert expires within ${min_days} days" + else + echo "OK: $cert valid, key matches, >= ${min_days} days remaining" + fi +} + +check_remote() { + endpoint="$1" + host="${endpoint%%:*}" + port="${endpoint#*:}" + [ "$port" = "$host" ] && port=443 + + echo "Checking remote endpoint: ${host}:${port}" + + probe="$(printf '' | openssl s_client -connect "${host}:${port}" \ + -servername "$host" -verify_hostname "$host" 2>/dev/null)" || { + fail "${host}:${port} TLS handshake failed" + return + } + + if ! printf '%s\n' "$probe" | grep -q "Verify return code: 0 (ok)"; then + code="$(printf '%s\n' "$probe" | sed -n 's/^ *Verify return code: //p' | head -n1)" + fail "${host}:${port} certificate verification failed (${code:-no verify result})" + return + fi + + if ! printf '%s\n' "$probe" | openssl x509 -noout -checkend "$min_seconds" >/dev/null 2>&1; then + fail "${host}:${port} certificate expires within ${min_days} days" + else + echo "OK: ${host}:${port} chain verified, >= ${min_days} days remaining" + fi +} + +ran_any=0 + +if [ "$#" -ge 1 ]; then + ran_any=1 + check_local "$1" +fi + +if [ -n "${CERT_TEST_REMOTE_ENDPOINTS:-}" ]; then + ran_any=1 + for endpoint in $CERT_TEST_REMOTE_ENDPOINTS; do + check_remote "$endpoint" + done +fi + +if [ "$ran_any" = "0" ]; then + echo "Usage: $0 []" >&2 + echo "Set CERT_TEST_REMOTE_ENDPOINTS=\"host[:port] ...\" for remote checks." >&2 + exit 1 +fi + +if [ "$failures" -gt 0 ]; then + echo "Certificate test finished with ${failures} failure(s)." >&2 + exit 1 +fi + +echo "Certificate test passed." diff --git a/roles/sys-ctl-mtn-cert-deploy/meta/services.yml b/roles/sys-ctl-mtn-cert-deploy/meta/services.yml index a3e2ebd682..0e19e2db9c 100644 --- a/roles/sys-ctl-mtn-cert-deploy/meta/services.yml +++ b/roles/sys-ctl-mtn-cert-deploy/meta/services.yml @@ -3,6 +3,6 @@ cert-deploy: lifecycle: beta required_by: compose: - roles: [web-app-mailu] + roles: [web-app-stalwart, web-app-mailu] swarm: - roles: [web-app-mailu] + roles: [web-app-stalwart, web-app-mailu] diff --git a/roles/sys-ctl-mtn-cert-deploy/tasks/00_core.yml b/roles/sys-ctl-mtn-cert-deploy/tasks/00_core.yml index aa1e4b2f26..16f840aaed 100644 --- a/roles/sys-ctl-mtn-cert-deploy/tasks/00_core.yml +++ b/roles/sys-ctl-mtn-cert-deploy/tasks/00_core.yml @@ -20,6 +20,16 @@ --chdir {{ lookup('container', application_id, 'directories.instance') }} --project {{ (lookup('container', application_id, 'directories.instance') | regex_replace('/+$', '')) | basename }} +- name: "✅ Verify deployed certificate material" + ansible.builtin.script: + cmd: "{{ [ role_path, 'files/shell/verify.sh' ] | path_join }} {{ lookup('container', application_id, 'directories.instance') }}" + environment: + CERT_TEST_REMOTE_ENDPOINTS: >- + {{ cert_deploy_remote_endpoints + | default(lookup('env', 'CERT_TEST_REMOTE_ENDPOINTS'), true) }} + changed_when: false + when: MODE_DEBUG | bool + - name: 📂 Include dependency 'sys-ctl-alm-compose' include_role: name: sys-ctl-alm-compose diff --git a/roles/sys-front-tls-selfsigned/tasks/00_core.yml b/roles/sys-front-tls-selfsigned/tasks/00_core.yml index 2b835cfa9a..94f6ecfdab 100644 --- a/roles/sys-front-tls-selfsigned/tasks/00_core.yml +++ b/roles/sys-front-tls-selfsigned/tasks/00_core.yml @@ -110,7 +110,7 @@ {{ ( (_ss_cert_text.stdout | default('')) | - regex_findall('DNS:([^,\\s;]+)') | + regex_findall('DNS:([^,\s;]+)') | map('lower') | list | unique | sort ) if (_ss_cert_stat.stat.exists | default(false)) else diff --git a/roles/sys-svc-compose-ca/files/python/compose_ca.py b/roles/sys-svc-compose-ca/files/python/compose_ca.py index edac85d8bc..ef46f9d059 100755 --- a/roles/sys-svc-compose-ca/files/python/compose_ca.py +++ b/roles/sys-svc-compose-ca/files/python/compose_ca.py @@ -290,10 +290,17 @@ def docker_image_has_bin_sh( ImageMeta = tuple[bool, list[str], list[str], bool | None] -def _gather_one_image(image: str, *, cwd: Path, env: dict[str, str]) -> ImageMeta: +def _gather_one_image( + image: str, *, cwd: Path, env: dict[str, str], pull_if_absent: bool = True +) -> ImageMeta: rc, out, _err = run( ["docker", "image", "inspect", image], cwd=cwd, env=env, timeout=90 ) + if rc not in {0, TIMEOUT_RC} and pull_if_absent: + run(["docker", "pull", image], cwd=cwd, env=env, timeout=600, capture=False) + rc, out, _err = run( + ["docker", "image", "inspect", image], cwd=cwd, env=env, timeout=90 + ) if rc != 0: return (False, [], [], None if rc == TIMEOUT_RC else False) try: @@ -313,17 +320,28 @@ def _gather_one_image(image: str, *, cwd: Path, env: dict[str, str]) -> ImageMet def gather_image_meta( - images: list[str], *, cwd: Path, env: dict[str, str] + images: list[str], + *, + cwd: Path, + env: dict[str, str], + built_images: set[str] | None = None, ) -> dict[str, ImageMeta]: """Gather (exists, entrypoint, cmd, has_sh) per unique image, concurrently — the per-image docker reads dominate the inject and must not run serially under DiD latency or they blow the handler timeout for many-service apps.""" if not images: return {} + built = built_images or set() workers = min(8, len(images)) with ThreadPoolExecutor(max_workers=workers) as pool: futures = { - pool.submit(_gather_one_image, image, cwd=cwd, env=env): image + pool.submit( + _gather_one_image, + image, + cwd=cwd, + env=env, + pull_if_absent=image not in built, + ): image for image in images } return {futures[f]: f.result() for f in as_completed(futures)} @@ -581,6 +599,14 @@ def render_override( ), cwd=cwd, env=env, + built_images={ + svc["image"].strip() + for svc in services.values() + if isinstance(svc, dict) + and isinstance(svc.get("image"), str) + and svc["image"].strip() + and _has_build(svc) + }, ) for name, svc in services.items(): @@ -608,6 +634,7 @@ def render_override( environment["SSL_CERT_FILE"] = bundle_container environment["REQUESTS_CA_BUNDLE"] = bundle_container environment["CURL_CA_BUNDLE"] = bundle_container + environment["CA_TRUST_BUNDLE"] = bundle_container if extra_ca_host: extra_ca_container = str(Path(ca_container).parent / "ca-trust-extra.crt") volumes.append(f"{extra_ca_host}:{extra_ca_container}:ro") diff --git a/roles/sys-svc-compose/tasks/00_core.yml b/roles/sys-svc-compose/tasks/00_core.yml index 0d5e1afcc7..ab5af6efaa 100644 --- a/roles/sys-svc-compose/tasks/00_core.yml +++ b/roles/sys-svc-compose/tasks/00_core.yml @@ -52,3 +52,4 @@ - IS_SWARM_MODE - (storage.backend | default('local')) == 'nfs' - (groups['svc-storage-nfs-server'] | default([])) | length > 0 + - inventory_hostname not in (groups['svc-storage-nfs-server'] | default([])) diff --git a/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 b/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 index 45d0f66535..f703f5447d 100644 --- a/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 +++ b/roles/sys-svc-mail-msmtp/templates/msmtprc.conf.j2 @@ -1,6 +1,8 @@ {% set email = lookup('email') %} {% set ca_injected = sys_svc_mail_msmtp_in_container | bool and lookup('ca_injected', application_id) %} +{% set smtp_target = lookup('smtp_host', email, in_container=sys_svc_mail_msmtp_in_container) %} defaults +domain {{ DOMAIN_PRIMARY }} logfile ~/.msmtp.log timeout {{ email.timeout }} syslog {{ MODE_DEBUG | on_off }} @@ -11,8 +13,13 @@ tls {{ email.tls | on_off }} tls_trust_file {{ CA_TRUST.inject_bundle_container }} {% endif %} tls_host_override {{ email.host }} +{% if smtp_target.endswith('.onion') %} +{% set socks = (lookup('tor_socks_proxy') | regex_replace('^socks5h://', '')) if sys_svc_mail_msmtp_in_container | bool else lookup('tor_socks') %} +proxy_host {{ socks.split(':')[0] }} +proxy_port {{ socks.split(':')[1] }} +{% endif %} account system_email_no_reply -host {{ lookup('smtp_host', email, in_container=sys_svc_mail_msmtp_in_container) }} +host {{ smtp_target }} port {{ email.port }} from {{ email['from'] }} {% if (email.auth | bool) %} diff --git a/roles/sys-svc-mail/README.md b/roles/sys-svc-mail/README.md index fc3f49fc3c..6bfce1925c 100644 --- a/roles/sys-svc-mail/README.md +++ b/roles/sys-svc-mail/README.md @@ -5,9 +5,9 @@ The `sys-svc-mail` role acts as the **central mail orchestration layer** in the Infinito.Nexus stack. It wires together: -- [Mailu](https://mailu.io/) as a full-featured mail server (when available), +- [Stalwart](https://stalw.art/) as a full-featured mail server (when available), - [msmtp](https://marlam.de/msmtp/) as a lightweight sendmail-compatible SMTP client, and -- an optional local SMTP relay (Postfix) for hosts **without** Mailu. +- an optional local SMTP relay (Postfix) for hosts **without** Stalwart. For more background on the underlying protocol, see [Simple Mail Transfer Protocol (SMTP) on Wikipedia](https://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol). @@ -15,16 +15,15 @@ For more background on the underlying protocol, see [Simple Mail Transfer Protoc This role provides a **unified mail setup** for your hosts: -- If the host is part of the `web-app-mailu` group, it: - - checks the reachability of the Mailu endpoint, - - triggers Mailu startup via the Infinito.Nexus helper (`utils/load/app.yml`), - - and prepares the system to send emails through Mailu using the `sys-svc-mail-msmtp` role. +- If the host is part of the active mail provider's group (`MAIL_PROVIDER`), it: + - asserts the provider endpoint was preloaded (submission token present), + - and prepares the system to send emails through the provider using the `sys-svc-mail-msmtp` role. -- If the host is **not** running Mailu, it: +- If the host is **not** running Stalwart, it: - optionally configures a local SMTP relay via `sys-svc-mail-smtp` (Postfix on `localhost:25`), - and still configures `msmtp` as a sendmail-compatible client. -This makes `sys-svc-mail` the canonical entrypoint for “mail capabilities” on a node, abstracting away whether the actual delivery happens via Mailu or a local relay. +This makes `sys-svc-mail` the canonical entrypoint for “mail capabilities” on a node, abstracting away whether the actual delivery happens via Stalwart or a local relay. ## Purpose @@ -32,25 +31,24 @@ The main purpose of this role is to: - Provide a **consistent mail-sending interface** for all hosts in the Infinito.Nexus ecosystem. - Automatically choose between: - - **Mailu-backed delivery** (with authentication tokens), or + - **Stalwart-backed delivery** (with authentication tokens), or - a **local SMTP relay on localhost**, - depending on the presence of `web-app-mailu` in the host’s groups. + depending on the presence of `web-app-stalwart` in the host’s groups. - Ensure that system services and applications can always send notifications (e.g. health checks, alerts, job results) without each role having to care about the underlying mail plumbing. ## Features -- 🔄 **Mailu Integration (when available)** - - Checks Mailu reachability using Ansible’s `uri` module. - - Triggers Mailu startup via `utils/load/app.yml`. - - Ensures handlers are flushed/reset via `utils/load/handlers.yml`. +- 🔄 **Mail-provider integration (when available)** + - Asserts the active provider's submission token was provisioned before configuring external mail. + - Routes outbound mail through the provider via `sys-svc-mail-msmtp`. - 💡 **Smart Fallback to Localhost** - - If no `web-app-mailu` is present, the role can configure a local Postfix-based SMTP relay via `sys-svc-mail-smtp`. + - If no `web-app-stalwart` is present, the role can configure a local Postfix-based SMTP relay via `sys-svc-mail-smtp`. - Combined with `sys-svc-mail-msmtp`, this enables sending mail via `localhost:25` without additional configuration in other roles. - 📨 **msmtp Client Configuration** - Delegates installation and configuration of msmtp to `sys-svc-mail-msmtp`. - - Supports both authenticated Mailu delivery and unauthenticated localhost-based delivery. + - Supports both authenticated Stalwart delivery and unauthenticated localhost-based delivery. - 🧩 **Composable Design** - Uses internal `run_once_*` flags to avoid repeated setup. @@ -59,7 +57,7 @@ The main purpose of this role is to: ## Further Resources - Mail server: - - Mailu: + - Stalwart: - SMTP (protocol): - SMTP client: - msmtp: diff --git a/roles/sys-svc-mail/meta/main.yml b/roles/sys-svc-mail/meta/main.yml index 3fb0667586..bfc48ba24f 100644 --- a/roles/sys-svc-mail/meta/main.yml +++ b/roles/sys-svc-mail/meta/main.yml @@ -1,7 +1,7 @@ --- galaxy_info: author: Kevin Veen-Birkenbach - description: Central mail orchestration layer routing host mail through Mailu, msmtp, or an optional local SMTP relay + description: Central mail orchestration role for Infinito.Nexus, integrating Email SaaS, msmtp, and an optional local SMTP relay license: Infinito.Nexus Community License (Non-Commercial) company: | Kevin Veen-Birkenbach diff --git a/roles/sys-svc-mail/tasks/00_core.yml b/roles/sys-svc-mail/tasks/00_core.yml index 8e6d1848b1..63dbc0b8fe 100644 --- a/roles/sys-svc-mail/tasks/00_core.yml +++ b/roles/sys-svc-mail/tasks/00_core.yml @@ -6,17 +6,20 @@ name: "sys-hostname" when: not (run_once_sys_hostname | default(false) | bool) -- name: "✅ Assert Mailu was preloaded before configuring external mail" +- name: "✅ Assert the mail provider was preloaded before configuring external mail" ansible.builtin.assert: that: - - run_once_web_app_mailu is defined - - (lookup('users', 'no-reply').tokens | default({})).get('web-app-mailu', '') | length > 0 + - >- + lookup('vars', 'run_once_' ~ (MAIL_PROVIDER | replace('-', '_')), + default='__missing__') != '__missing__' + - (lookup('users', 'no-reply').tokens | default({})).get(MAIL_PROVIDER, '') | length > 0 fail_msg: >- - Mailu must be preloaded by sys-service-loader before sys-svc-mail runs. - Missing run_once_web_app_mailu or missing users['no-reply'].tokens['web-app-mailu'] - indicates that a service was loaded outside the constructor service-loader path. + {{ MAIL_PROVIDER }} must be preloaded by sys-service-loader before + sys-svc-mail runs. Missing run_once_{{ MAIL_PROVIDER | replace('-', '_') }} + or missing users['no-reply'].tokens['{{ MAIL_PROVIDER }}'] indicates that + a service was loaded outside the constructor service-loader path. when: - - "'web-app-mailu' in group_names" + - MAIL_PROVIDER in group_names - MODE_ASSERT_LOCAL | bool - name: 🔧 Setup emails via localhost diff --git a/roles/sys-svc-rdbms/tasks/dedicated.yml b/roles/sys-svc-rdbms/tasks/dedicated.yml index 9d9b87a6d0..c3039c0f29 100644 --- a/roles/sys-svc-rdbms/tasks/dedicated.yml +++ b/roles/sys-svc-rdbms/tasks/dedicated.yml @@ -39,3 +39,4 @@ when: - "'svc-bkp-volume-2-local' in group_names" - IS_STACK_HOST | bool + - lookup('database', _database_consumer_id, 'managed') | bool diff --git a/roles/test-e2e-playwright/files/personas/utils/index.js b/roles/test-e2e-playwright/files/personas/utils/index.js index 1b5747342e..0391e3410d 100644 --- a/roles/test-e2e-playwright/files/personas/utils/index.js +++ b/roles/test-e2e-playwright/files/personas/utils/index.js @@ -16,6 +16,7 @@ const env = require("./env"); const onion = require("./onion"); const tls = require("./tls"); const keycloak = require("./keycloak"); +const mapache = require("./mapache"); const logout = require("./logout"); const landing = require("./landing"); const csp = require("./csp"); @@ -23,12 +24,14 @@ const interaction = require("./interaction"); const dotenv = require("./dotenv"); const dom = require("./dom"); const seaweedfs = require("./seaweedfs"); +const mailbox = require("./mailbox"); module.exports = { ...env, ...onion, ...tls, ...keycloak, + ...mapache, ...logout, ...landing, ...csp, @@ -36,4 +39,5 @@ module.exports = { ...dotenv, ...dom, ...seaweedfs, + ...mailbox, }; diff --git a/roles/test-e2e-playwright/files/personas/utils/mailbox.js b/roles/test-e2e-playwright/files/personas/utils/mailbox.js new file mode 100644 index 0000000000..6c3876d1e6 --- /dev/null +++ b/roles/test-e2e-playwright/files/personas/utils/mailbox.js @@ -0,0 +1,87 @@ +/** + * Read a message back out of the recipient's mailbox, through the mail + * provider's webmail. + * + * Outbound mail configuration fails in ways the sender never notices: an + * implicit-TLS port announced as STARTTLS, AUTH offered on a listener that has + * none, or a relay the application can reach but the provider rejects. In every + * one of those the send call still returns success. The only proof that a + * setting is right is the message arriving, so these helpers exist for any role + * whose configuration decides how mail leaves it. + * + * `webmailSsoLogin(page, username, password, baseUrl)` + * Drives the provider's webmail login. The webmail redirects to Keycloak on + * its own (`oauth_login_redirect`), so this only has to click an SSO control + * when one is rendered, then fill the Keycloak form. A caller that already + * holds a Keycloak session never sees a form — the redirect answers straight + * back into the mailbox — so the helper waits for whichever of the two + * surfaces arrives instead of hanging on the form. + * + * `waitForEmailInMailbox(page, baseUrl, subject, timeout)` + * Polls INBOX and Junk for a message whose subject matches, and returns its + * row. Junk is not a fallback for a broken setup: a test domain publishes no + * mail-auth DNS, so a correctly delivered message is legitimately filed + * there. Folders are switched by `_mbox` URL, because clicking a folder link + * hangs when the row it points at is not actionable yet. + */ + +const { performKeycloakLoginForm } = require("./keycloak"); +const { gotoOnion } = require("./env"); +const { resolveTimeout } = require("../../timeouts"); + +const MAIL_UI = "#messagelist, .compose, a[href*='_action=compose'], .toolbar"; +const SSO_CONTROL = /sso|single sign.?on|login with|openid/i; + +const MAILBOXES = ["INBOX", "Junk Mail"]; + +async function webmailSsoLogin(page, username, password, baseUrl) { + await gotoOnion(page, `${baseUrl}/`); + const ssoButton = page + .getByRole("button", { name: SSO_CONTROL }) + .or(page.getByRole("link", { name: SSO_CONTROL })); + if (await ssoButton.first().isVisible().catch(() => false)) { + await ssoButton.first().click({ timeout: resolveTimeout(30_000) }); + } + + const mailUi = page.locator(MAIL_UI).first(); + const usernameField = page + .getByRole("textbox", { name: /username|email/i }) + .or(page.locator("input[name='username'], input#username")) + .first(); + + await Promise.any([ + mailUi.waitFor({ state: "visible", timeout: resolveTimeout(60_000) }), + usernameField.waitFor({ state: "visible", timeout: resolveTimeout(60_000) }), + ]); + if (await usernameField.isVisible().catch(() => false)) { + await performKeycloakLoginForm(page, username, password); + } + await mailUi.waitFor({ state: "visible", timeout: resolveTimeout(60_000) }); +} + +function messageRow(page, subject) { + return page + .locator("#messagelist tbody tr, table.messagelist tbody tr") + .filter({ hasText: subject }) + .first(); +} + +async function waitForEmailInMailbox(page, baseUrl, subject, timeout = resolveTimeout(90_000)) { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + for (const mailbox of MAILBOXES) { + await gotoOnion(page, `${baseUrl}/?_task=mail&_mbox=${encodeURIComponent(mailbox)}`, { + waitUntil: "domcontentloaded", + }).catch(() => {}); + await page.waitForTimeout(resolveTimeout(2_000)); + if (await messageRow(page, subject).isVisible().catch(() => false)) { + return messageRow(page, subject); + } + } + } + throw new Error( + `Timed out waiting for a message matching "${subject}" in ${MAILBOXES.join(" / ")}` + ); +} + +module.exports = { webmailSsoLogin, waitForEmailInMailbox }; diff --git a/roles/test-e2e-playwright/files/personas/utils/mapache.js b/roles/test-e2e-playwright/files/personas/utils/mapache.js new file mode 100644 index 0000000000..76ca41ce59 --- /dev/null +++ b/roles/test-e2e-playwright/files/personas/utils/mapache.js @@ -0,0 +1,18 @@ +/** + * The OIDC-only persona. + * + * `MAPACHE` + * A person the administrator onboards in Keycloak after the deploy, + * whom no application pre-provisions. Keycloak's spec provisions it + * once; consumers prove they create the account on first OIDC login. + */ + +const MAPACHE = { + username: "mapache", + firstName: "Infinito", + lastName: "Mapache", +}; + +module.exports = { + MAPACHE, +}; diff --git a/roles/test-e2e-playwright/filter_plugins/discover_playwright_roles.py b/roles/test-e2e-playwright/filter_plugins/discover_playwright_roles.py index ae33f4b790..7692785c2f 100644 --- a/roles/test-e2e-playwright/filter_plugins/discover_playwright_roles.py +++ b/roles/test-e2e-playwright/filter_plugins/discover_playwright_roles.py @@ -6,6 +6,11 @@ from typing import TYPE_CHECKING from ansible.errors import AnsibleFilterError +from utils.roles.meta_lookup import get_role_provides +from utils.roles.order import build_dependency_graph, topological_sort + +SSO_SERVICE = "sso" + if TYPE_CHECKING: from collections.abc import Iterable @@ -33,6 +38,40 @@ def _to_role_set(raw: Iterable[str] | str | None, var_name: str) -> set[str]: ) from exc +def _in_test_order(roles_dir: Path, roles: list[str]) -> list[str]: + """Order *roles* for the Playwright stage. + + Every role is deployed before any spec runs, so the order only matters for + state one spec creates for another. The identity provider's specs provision + users that consumers log in with, so the role that ``provides: sso`` runs + first even when it deploys late (Keycloak runs after its mail provider); + the rest follow the deploy order resolved by ``utils.roles.order``. + + Args: + roles_dir: directory holding the role folders. + roles: role names to order, alphabetically sorted. + + Returns: + The same roles: the sso provider first, then providers before their + consumers; roles outside the resolved graph keep their alphabetical + order at the end. + """ + graph, in_degree, meta = build_dependency_graph(roles_dir) + position = { + role: index + for index, role in enumerate(topological_sort(graph, in_degree, meta)) + } + return sorted( + roles, + key=lambda role: ( + get_role_provides((roles_dir / role).resolve(), role_name=role) + != SSO_SERVICE, + position.get(role, len(position)), + role, + ), + ) + + def discover_playwright_roles( playbook_dir: str, only_roles: Iterable[str] | str | None = None, @@ -52,7 +91,7 @@ def discover_playwright_roles( role_name = env_file.parents[1].name found.append(role_name) - uniq = sorted(set(found)) + uniq = _in_test_order(base, sorted(set(found))) if only: uniq = [role for role in uniq if role in only] diff --git a/roles/web-app-akaunting/meta/services.yml b/roles/web-app-akaunting/meta/services.yml index f91e678808..68a3855f13 100644 --- a/roles/web-app-akaunting/meta/services.yml +++ b/roles/web-app-akaunting/meta/services.yml @@ -13,8 +13,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — DB engine, covered by role-local integration tests mariadb: bond: 1 @@ -64,6 +64,7 @@ akaunting: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta cpus: "0.5" diff --git a/roles/web-app-akaunting/templates/compose.yml.j2 b/roles/web-app-akaunting/templates/compose.yml.j2 index 854f6ac2f9..4c9cf13b83 100644 --- a/roles/web-app-akaunting/templates/compose.yml.j2 +++ b/roles/web-app-akaunting/templates/compose.yml.j2 @@ -12,6 +12,7 @@ {{ lookup('container_ports', ['akaunting', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} diff --git a/roles/web-app-baserow/meta/services.yml b/roles/web-app-baserow/meta/services.yml index c764ab99b0..72fed5e068 100644 --- a/roles/web-app-baserow/meta/services.yml +++ b/roles/web-app-baserow/meta/services.yml @@ -13,8 +13,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" sso: bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" @@ -85,6 +85,7 @@ baserow: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu - web-svc-seaweedfs lifecycle: beta diff --git a/roles/web-app-baserow/templates/compose.yml.j2 b/roles/web-app-baserow/templates/compose.yml.j2 index e92300d63c..223ec85874 100644 --- a/roles/web-app-baserow/templates/compose.yml.j2 +++ b/roles/web-app-baserow/templates/compose.yml.j2 @@ -11,6 +11,7 @@ {{ lookup('compose_only', 'container_name', BASEROW_CONTAINER) }} {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('container_ports', ['baserow', 'http', DOCKER_BIND_HOST]) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} diff --git a/roles/web-app-bigbluebutton/meta/services.yml b/roles/web-app-bigbluebutton/meta/services.yml index 765c1aead3..b9c56e7533 100644 --- a/roles/web-app-bigbluebutton/meta/services.yml +++ b/roles/web-app-bigbluebutton/meta/services.yml @@ -22,8 +22,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag: shared provider entity; the app itself is covered by this role's own Playwright spec bigbluebutton: # TODO: make this role arm64-capable so the declaration below can go diff --git a/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 b/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 index 38b3bdebeb..53ced3bb76 100644 --- a/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 +++ b/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 @@ -3,6 +3,7 @@ services: {{ service_name }}: {{ lookup('container_image', application_id, service_name) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {% if BBB_COLLABORA_ENABLED | bool %} {{ lookup('depends_on', ['redis', 'etherpad', 'bbb-pads']) }} {% endif %} diff --git a/roles/web-app-bluesky/meta/services.yml b/roles/web-app-bluesky/meta/services.yml index 740f522320..f2d051e516 100644 --- a/roles/web-app-bluesky/meta/services.yml +++ b/roles/web-app-bluesky/meta/services.yml @@ -14,8 +14,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: dynamic-flag - LDAP is disabled for this role; the literal false is intentional, not group-gated. ldap: enabled: false @@ -140,6 +140,7 @@ bluesky: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta min_storage: 1GB diff --git a/roles/web-app-bluesky/templates/compose.yml.j2 b/roles/web-app-bluesky/templates/compose.yml.j2 index 96f723be50..1951fbe3c2 100644 --- a/roles/web-app-bluesky/templates/compose.yml.j2 +++ b/roles/web-app-bluesky/templates/compose.yml.j2 @@ -8,6 +8,7 @@ {{ lookup('container_ports', ['api', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_healthcheck', 'api') | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% if BLUESKY_WEB_ENABLED %} @@ -23,6 +24,7 @@ {{ lookup('compose_restart') }} {{ lookup('container_ports', ['web', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = BLUESKY_BROKER_SERVICE %} @@ -56,6 +58,7 @@ retries: 3 start_period: 10m start_interval: 5s + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% endif %} diff --git a/roles/web-app-bluesky/vars/main.yml b/roles/web-app-bluesky/vars/main.yml index faa80c1fdf..9771054275 100644 --- a/roles/web-app-bluesky/vars/main.yml +++ b/roles/web-app-bluesky/vars/main.yml @@ -30,7 +30,7 @@ BLUESKY_API_DOMAIN: "{{ lookup('domains', application_id).api }}" BLUESKY_API_PORT: "{{ lookup('config', 'web-app-bluesky', 'services.api.ports.local.http') }}" BLUESKY_PDS_INTERNAL_PORT: "{{ lookup('config', application_id, 'services.api.ports.internal.http') }}" -BLUESKY_PDS_HANDLE_DOMAIN: "{{ DOMAIN_PRIMARY | regex_replace('\\.(example|test|invalid|local|localhost|arpa|alt|internal|onion)$', '.app') }}" +BLUESKY_PDS_HANDLE_DOMAIN: "{{ DOMAIN_PRIMARY | regex_replace('\\.test$', '.app') }}" BLUESKY_LDAP_ENABLED: "{{ lookup('config', application_id, 'services.ldap.enabled') }}" diff --git a/roles/web-app-bookwyrm/meta/services.yml b/roles/web-app-bookwyrm/meta/services.yml index 3192ec7618..724085c62d 100644 --- a/roles/web-app-bookwyrm/meta/services.yml +++ b/roles/web-app-bookwyrm/meta/services.yml @@ -24,8 +24,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — DB engine, covered by role-local integration tests postgres: bond: 1 @@ -75,6 +75,7 @@ bookwyrm: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta cpus: "0.5" diff --git a/roles/web-app-bookwyrm/templates/compose.yml.j2 b/roles/web-app-bookwyrm/templates/compose.yml.j2 index d475a9e831..4afb6d4d61 100644 --- a/roles/web-app-bookwyrm/templates/compose.yml.j2 +++ b/roles/web-app-bookwyrm/templates/compose.yml.j2 @@ -24,6 +24,7 @@ {{ lookup('container_depends_on', application_id, extra={ 'migrate': {'condition': 'service_completed_successfully'}, }) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'worker' %} @@ -37,6 +38,7 @@ {{ lookup('container_depends_on', application_id, extra={ 'migrate': {'condition': 'service_completed_successfully'}, }) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'migrate' %} @@ -52,6 +54,7 @@ ' {{ lookup('compose_only', 'pull_policy', 'never') }} {{ lookup('container_depends_on', application_id) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('compose_volumes') }} diff --git a/roles/web-app-checkmk/meta/services.yml b/roles/web-app-checkmk/meta/services.yml index ab52dec53c..49cdf97c31 100644 --- a/roles/web-app-checkmk/meta/services.yml +++ b/roles/web-app-checkmk/meta/services.yml @@ -31,8 +31,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" css: bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" diff --git a/roles/web-app-decidim/meta/services.yml b/roles/web-app-decidim/meta/services.yml index 7e3aa7cf02..c43ecbbf69 100644 --- a/roles/web-app-decidim/meta/services.yml +++ b/roles/web-app-decidim/meta/services.yml @@ -19,8 +19,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — DB engine, covered by role-local integration tests postgres: bond: 1 diff --git a/roles/web-app-discourse/meta/services.yml b/roles/web-app-discourse/meta/services.yml index 51a2a06ede..dbe5f066b9 100644 --- a/roles/web-app-discourse/meta/services.yml +++ b/roles/web-app-discourse/meta/services.yml @@ -33,8 +33,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — DB engine, covered by role-local integration tests postgres: bond: 1 diff --git a/roles/web-app-discourse/templates/config.yml.j2 b/roles/web-app-discourse/templates/config.yml.j2 index 6df9842298..6addb54c9e 100644 --- a/roles/web-app-discourse/templates/config.yml.j2 +++ b/roles/web-app-discourse/templates/config.yml.j2 @@ -74,7 +74,7 @@ env: DISCOURSE_SMTP_ADDRESS: {{ email.host }} DISCOURSE_SMTP_PORT: {{ email.port }} DISCOURSE_SMTP_USER_NAME: {{ email.username }} - DISCOURSE_SMTP_PASSWORD: {{ email.password }} + DISCOURSE_SMTP_PASSWORD: {{ email.password | to_json }} DISCOURSE_SMTP_ENABLE_START_TLS: {{ email.start_tls }} DISCOURSE_SMTP_FORCE_TLS: {{ email.tls }} DISCOURSE_SMTP_DOMAIN: {{ email.domain }} @@ -164,3 +164,4 @@ docker_args: - --add-host={{ extra_host }} {% endfor %} - --name={{ DISCOURSE_CONTAINER }} +{{ lookup('container_extra_hosts', docker_flags=true) }} diff --git a/roles/web-app-erpnext/README.md b/roles/web-app-erpnext/README.md index 3ba86d7f08..5872ad6909 100644 --- a/roles/web-app-erpnext/README.md +++ b/roles/web-app-erpnext/README.md @@ -6,7 +6,7 @@ ## Overview -This role deploys ERPNext as an Infinito.Nexus web app using the upstream `frappe/erpnext` single-image multi-role pattern from [frappe_docker](https://github.com/frappe/frappe_docker): one Docker image, different commands per container (backend gunicorn, frontend nginx, websocket socketio, scheduler, two queue workers, plus a one-shot configurator). MariaDB and Redis come from the central `svc-db-mariadb` and `svc-db-redis` providers — the three Frappe Redis logical roles (`cache`, `queue`, `socketio`) share one central Redis instance via DB-number split (0 / 1 / 2). Authentication uses Frappe's built-in Social Login Key against the shared Keycloak OIDC client without an oauth2-proxy sidecar; LDAP federation and outbound SMTP via Mailu are wired when their providers are present. +This role deploys ERPNext as an Infinito.Nexus web app using the upstream `frappe/erpnext` single-image multi-role pattern from [frappe_docker](https://github.com/frappe/frappe_docker): one Docker image, different commands per container (backend gunicorn, frontend nginx, websocket socketio, scheduler, two queue workers, plus a one-shot configurator). MariaDB and Redis come from the central `svc-db-mariadb` and `svc-db-redis` providers — the three Frappe Redis logical roles (`cache`, `queue`, `socketio`) share one central Redis instance via DB-number split (0 / 1 / 2). Authentication uses Frappe's built-in Social Login Key against the shared Keycloak OIDC client without an oauth2-proxy sidecar; LDAP federation and outbound SMTP via Stalwart are wired when their providers are present. ## Addons @@ -25,7 +25,7 @@ Both name a Frappe doctype the image already ships, and the per-addon hooks [`ta - **Direct OIDC SSO** — Frappe's Social Login Key talks to the shared Keycloak client; redirect URI auto-registered via the `redirect_uris` filter (no per-app Keycloak entry needed). - **Keycloak group → Frappe role mapping** — three tiers: `roles/web-app-erpnext/administrator` → `System Manager`, `roles/web-app-erpnext/manager` → `{Sales, Purchase, Stock, Accounts} User`, default → `Customer`. Reconciled on each login by a hook reading the persisted map. - **LDAP federation** — when `svc-db-openldap` is present, Frappe's LDAP Settings doctype is auto-configured. End-to-end LDAP login (auto-create Frappe user from LDAP bind) is deferred to a follow-up requirement; v1 ships the Settings doctype only. The LDAP-login Playwright specs are intentionally absent for v1. -- **Outbound mail (v1)** — when `web-app-mailu` is present, Frappe's outbound Email Account is auto-configured against the central SMTP endpoint. **IMAP inbound (mail-to-Communication) is deferred to a follow-up requirement** (no precedent in the repo for auto-provisioning role-owned Mailu mailboxes yet). +- **Outbound mail (v1)** — when `web-app-stalwart` is present, Frappe's outbound Email Account is auto-configured against the central SMTP endpoint. **IMAP inbound (mail-to-Communication) is deferred to a follow-up requirement** (no precedent in the repo for auto-provisioning role-owned Stalwart mailboxes yet). - **Wizard bypass** — first deploy seeds the API-bot user, marks `setup_complete=1` in System Settings, and skips the Frappe setup wizard entirely. ## Image & version policy diff --git a/roles/web-app-erpnext/meta/services.yml b/roles/web-app-erpnext/meta/services.yml index 65e39164ce..be86d8e07e 100644 --- a/roles/web-app-erpnext/meta/services.yml +++ b/roles/web-app-erpnext/meta/services.yml @@ -11,8 +11,8 @@ ldap: shared: "{{ 'svc-db-openldap' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" logout: bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" @@ -79,6 +79,7 @@ erpnext: service_key: frontend run_after: - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta cpus: "2.0" diff --git a/roles/web-app-erpnext/templates/compose.yml.j2 b/roles/web-app-erpnext/templates/compose.yml.j2 index 1ddd9a834a..e26c6b01b9 100644 --- a/roles/web-app-erpnext/templates/compose.yml.j2 +++ b/roles/web-app-erpnext/templates/compose.yml.j2 @@ -11,6 +11,7 @@ {{ lookup('depends_on', {'configurator': 'service_completed_successfully'}) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'queue-short' %} @@ -21,6 +22,7 @@ command: ["bash", "-c", "{{ ERPNEXT_CONFIG_READY_GATE }} exec bench worker --queue short,default"] {{ lookup('depends_on', {'backend': 'service_started'}) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'queue-long' %} @@ -31,6 +33,7 @@ command: ["bash", "-c", "{{ ERPNEXT_CONFIG_READY_GATE }} exec bench worker --queue long,default,short"] {{ lookup('depends_on', {'backend': 'service_started'}) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'frontend' %} @@ -52,6 +55,7 @@ {{ lookup('container_ports', ['erpnext', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'websocket' %} @@ -62,6 +66,7 @@ command: ["bash", "-c", "{{ ERPNEXT_CONFIG_READY_GATE }} exec node /home/frappe/frappe-bench/apps/frappe/socketio.js"] {{ lookup('depends_on', {'configurator': 'service_completed_successfully'}) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'scheduler' %} @@ -72,6 +77,7 @@ command: ["bash", "-c", "{{ ERPNEXT_CONFIG_READY_GATE }} exec bench schedule"] {{ lookup('depends_on', {'backend': 'service_started'}) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'configurator' %} @@ -95,6 +101,7 @@ bench set-config -g host_name "{{ lookup('tls', application_id, 'url.base') | regex_replace('/+$', '') }}"; {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('compose_volumes') }} diff --git a/roles/web-app-espocrm/meta/services.yml b/roles/web-app-espocrm/meta/services.yml index c182dc90b9..e2b2bb99c2 100644 --- a/roles/web-app-espocrm/meta/services.yml +++ b/roles/web-app-espocrm/meta/services.yml @@ -23,8 +23,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" mariadb: bond: 1 enabled: true # nocheck: dynamic-flag @@ -60,6 +60,7 @@ espocrm: start_period: 15m run_after: - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta daemon: diff --git a/roles/web-app-espocrm/templates/env.j2 b/roles/web-app-espocrm/templates/env.j2 index d01aad6cde..51119580a5 100644 --- a/roles/web-app-espocrm/templates/env.j2 +++ b/roles/web-app-espocrm/templates/env.j2 @@ -32,7 +32,7 @@ ESPOCRM_CONFIG_SMTP_PORT={{ email.port }} ESPOCRM_CONFIG_SMTP_SECURITY={{ "TLS" if email.start_tls else "SSL"}} ESPOCRM_CONFIG_SMTP_AUTH=true ESPOCRM_CONFIG_SMTP_USERNAME={{ lookup('users', 'contact').email }} -ESPOCRM_CONFIG_SMTP_PASSWORD={{ lookup('users', 'contact').get('tokens', {}).get('web-app-mailu', '') | dotenv_quote }} +ESPOCRM_CONFIG_SMTP_PASSWORD={{ lookup('users', 'contact').get('tokens', {}).get(MAIL_PROVIDER, '') | dotenv_quote }} ESPOCRM_CONFIG_OUTBOUND_EMAIL_FROM_NAME={{ lookup('config', application_id, 'services.espocrm.email.from_name')}} ESPOCRM_CONFIG_OUTBOUND_EMAIL_FROM_ADDRESS={{ lookup('users', 'contact').email }} {% endif %} diff --git a/roles/web-app-fider/meta/services.yml b/roles/web-app-fider/meta/services.yml index f8aa26b643..5a70babb45 100644 --- a/roles/web-app-fider/meta/services.yml +++ b/roles/web-app-fider/meta/services.yml @@ -6,8 +6,8 @@ sso: flavor: oidc email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" logout: bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" @@ -57,6 +57,7 @@ fider: retries: 5 run_after: - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta # nocheck: dynamic-flag diff --git a/roles/web-app-fider/templates/compose.yml.j2 b/roles/web-app-fider/templates/compose.yml.j2 index 8be0b699ac..db25f356b9 100644 --- a/roles/web-app-fider/templates/compose.yml.j2 +++ b/roles/web-app-fider/templates/compose.yml.j2 @@ -8,6 +8,7 @@ {{ lookup('container_ports', ['fider', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} diff --git a/roles/web-app-flowise/meta/services.yml b/roles/web-app-flowise/meta/services.yml index 42a62e31bc..0a6448c40e 100644 --- a/roles/web-app-flowise/meta/services.yml +++ b/roles/web-app-flowise/meta/services.yml @@ -89,6 +89,7 @@ flowise: run_after: - web-app-keycloak - web-app-matomo + - web-app-stalwart - web-app-mailu lifecycle: beta # nocheck: dynamic-flag @@ -115,8 +116,8 @@ javascript: shared: true email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" prometheus: bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-flowise/templates/compose.yml.j2 b/roles/web-app-flowise/templates/compose.yml.j2 index 7030529c11..69bb8dfd29 100644 --- a/roles/web-app-flowise/templates/compose.yml.j2 +++ b/roles/web-app-flowise/templates/compose.yml.j2 @@ -17,6 +17,7 @@ {{ lookup('container_depends_on', application_id) | indent(4) }} {{ lookup('container_ports', ['flowise', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('compose_networks') }} diff --git a/roles/web-app-friendica/meta/services.yml b/roles/web-app-friendica/meta/services.yml index 4e92b1bd5d..f87f3b920c 100644 --- a/roles/web-app-friendica/meta/services.yml +++ b/roles/web-app-friendica/meta/services.yml @@ -86,8 +86,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag, DB engine, covered by role-local integration tests mariadb: bond: 1 diff --git a/roles/web-app-friendica/templates/compose.yml.j2 b/roles/web-app-friendica/templates/compose.yml.j2 index a0b16ec6d4..fe9221e499 100644 --- a/roles/web-app-friendica/templates/compose.yml.j2 +++ b/roles/web-app-friendica/templates/compose.yml.j2 @@ -12,6 +12,7 @@ {{ lookup('container_ports', ['friendica', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} diff --git a/roles/web-app-funkwhale/meta/services.yml b/roles/web-app-funkwhale/meta/services.yml index 3cca650274..a6e7b88b74 100644 --- a/roles/web-app-funkwhale/meta/services.yml +++ b/roles/web-app-funkwhale/meta/services.yml @@ -24,6 +24,7 @@ funkwhale: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta cpus: "1.0" @@ -81,8 +82,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: shared # nocheck: dynamic-flag redis: diff --git a/roles/web-app-funkwhale/templates/compose.yml.j2 b/roles/web-app-funkwhale/templates/compose.yml.j2 index 97c9c3735f..32e3e824c9 100644 --- a/roles/web-app-funkwhale/templates/compose.yml.j2 +++ b/roles/web-app-funkwhale/templates/compose.yml.j2 @@ -10,6 +10,7 @@ {{ lookup('container_depends_on', application_id, extra={ 'migrate': {'condition': 'service_completed_successfully'}, }) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'celerybeat' %} @@ -21,6 +22,7 @@ {{ lookup('container_depends_on', application_id, extra={ 'migrate': {'condition': 'service_completed_successfully'}, }) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'api' %} @@ -36,6 +38,7 @@ {{ lookup('container_depends_on', application_id, extra={ 'migrate': {'condition': 'service_completed_successfully'}, }) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'front' %} @@ -48,6 +51,7 @@ - "NGINX_MAX_BODY_SIZE=100M" {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('container_ports', ['funkwhale', 'http', DOCKER_BIND_HOST]) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'typesense' %} @@ -68,6 +72,7 @@ {{ lookup('container_image', application_id, 'api') }} command: migrate {{ lookup('container_depends_on', application_id) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('compose_volumes') }} diff --git a/roles/web-app-fusiondirectory/meta/services.yml b/roles/web-app-fusiondirectory/meta/services.yml index 30cbbaf70f..d17dc630bd 100644 --- a/roles/web-app-fusiondirectory/meta/services.yml +++ b/roles/web-app-fusiondirectory/meta/services.yml @@ -19,8 +19,8 @@ tor: shared: true # nocheck: dynamic-flag email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" matomo: bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" @@ -62,6 +62,7 @@ fusiondirectory: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu - svc-db-openldap lifecycle: beta diff --git a/roles/web-app-gitea/meta/services.yml b/roles/web-app-gitea/meta/services.yml index 66f61c03c7..c05fc7f936 100644 --- a/roles/web-app-gitea/meta/services.yml +++ b/roles/web-app-gitea/meta/services.yml @@ -30,8 +30,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — DB engine, covered by role-local integration tests mariadb: bond: 1 @@ -82,6 +82,7 @@ gitea: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta # nocheck: playwright-service-flag diff --git a/roles/web-app-gitea/templates/compose.yml.j2 b/roles/web-app-gitea/templates/compose.yml.j2 index 1e251d7641..347f4090e2 100644 --- a/roles/web-app-gitea/templates/compose.yml.j2 +++ b/roles/web-app-gitea/templates/compose.yml.j2 @@ -10,6 +10,7 @@ ['gitea', 'ssh']) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% if GITEA_MCP_ENABLED | bool %} {{ GITEA_MCP_UPSTREAM_NETWORK }}: diff --git a/roles/web-app-gitlab/files/playwright/playwright.spec.js b/roles/web-app-gitlab/files/playwright/playwright.spec.js index 920c9aa488..dffc1526e3 100644 --- a/roles/web-app-gitlab/files/playwright/playwright.spec.js +++ b/roles/web-app-gitlab/files/playwright/playwright.spec.js @@ -2,6 +2,7 @@ const { test, expect } = require("@playwright/test"); const { resolveTimeout } = require("./timeouts"); const { + decodeDotenvQuotedValue, expectHstsWhenTls, gotoOnion, normalizeBaseUrl, @@ -9,11 +10,19 @@ const { runAdminFlow, runBiberFlow, runGuestFlow, + safeIsEnabled, + waitForEmailInMailbox, + webmailSsoLogin, } = require("./personas"); test.use({ ignoreHTTPSErrors: true }); const appBaseUrl = normalizeBaseUrl(process.env.APP_BASE_URL || ""); const canonicalDomain = requireDotenvValue(process.env.CANONICAL_DOMAIN, "CANONICAL_DOMAIN"); +const webmailBaseUrl = normalizeBaseUrl(decodeDotenvQuotedValue(process.env.WEBMAIL_BASE_URL || "")); +const adminEmail = decodeDotenvQuotedValue(process.env.ADMIN_EMAIL || ""); +const adminUsername = decodeDotenvQuotedValue(process.env.ADMIN_USERNAME || ""); +const adminPassword = decodeDotenvQuotedValue(process.env.ADMIN_PASSWORD || ""); +const ssoEnabled = (process.env.SSO_SERVICE_ENABLED || "").toLowerCase() === "true"; test.beforeEach(async ({ page }) => { expect(appBaseUrl, "APP_BASE_URL must be set").toBeTruthy(); @@ -46,6 +55,42 @@ test("GitLab returns HTML content under canonical domain", async ({ request }) = require("./test-mcp-guest"); require("./test-integrations").register(); +// Outbound mail. `templates/config/smtp_settings.rb.j2` derives ActionMailer's +// `tls:` from the provider's own declaration, and tls/starttls are mutually +// exclusive there: implicit TLS on 465, or STARTTLS on the relay port, never +// both. Getting it wrong does not fail the send — GitLab queues the mail and +// reports success — so the only proof is reading the message back out of the +// recipient's mailbox. GitLab only mails an address it knows, and the OmniAuth +// sign-in is what creates the account, so the persona logs in once before asking. +test("gitlab: a password-reset request is delivered to the recipient's mailbox", async ({ page }) => { + test.skip(!safeIsEnabled("email"), "no mail provider in this deploy"); + test.skip(!ssoEnabled, "the administrator account is created by the first SSO login"); + test.skip(!webmailBaseUrl, "the active mail provider serves no webmail vhost to read from"); + expect(adminEmail, "ADMIN_EMAIL must be set").toBeTruthy(); + expect(adminPassword, "ADMIN_PASSWORD must be set").toBeTruthy(); + + await runAdminFlow(page); + + await gotoOnion(page, `${appBaseUrl}/users/password/new`); + await page.locator("input[type='email'], #user_email").first().fill(adminEmail); + await page.locator("input[type='submit'], button[type='submit']").first().click(); + await expect( + page.locator("body"), + "GitLab must confirm it accepted the reset request" + ).toContainText(/recovery link|password recovery|reset|instructions|receive/i, { + timeout: resolveTimeout(30_000), + }); + + const mailbox = await page.context().newPage(); + try { + await webmailSsoLogin(mailbox, adminUsername, adminPassword, webmailBaseUrl); + const row = await waitForEmailInMailbox(mailbox, webmailBaseUrl, "password", resolveTimeout(120_000)); + await expect(row, "the reset mail must reach the mailbox").toBeVisible(); + } finally { + await mailbox.close().catch(() => {}); + } +}); + // Persona scenarios. // Bodies live in the shared helper roles/test-e2e-playwright/files/personas // so every role's persona flow stays consistent. diff --git a/roles/web-app-gitlab/meta/services.yml b/roles/web-app-gitlab/meta/services.yml index 993709900e..be8163fa12 100644 --- a/roles/web-app-gitlab/meta/services.yml +++ b/roles/web-app-gitlab/meta/services.yml @@ -20,8 +20,8 @@ matomo: # nocheck: playwright-service-flag email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: shared # nocheck: dynamic-flag # nocheck: playwright-service-flag @@ -71,6 +71,7 @@ gitlab: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta webservice: diff --git a/roles/web-app-gitlab/templates/compose.yml.j2 b/roles/web-app-gitlab/templates/compose.yml.j2 index afe36be01c..c2e9cceeee 100644 --- a/roles/web-app-gitlab/templates/compose.yml.j2 +++ b/roles/web-app-gitlab/templates/compose.yml.j2 @@ -15,6 +15,7 @@ GITLAB_MIGRATIONS_SERVICE_NAME: {'condition': 'service_completed_successfully'}, }) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks', provider_self_alias=False) | indent(4) }} {% set service_name = GITLAB_WORKHORSE_SERVICE_NAME %} @@ -52,6 +53,7 @@ GITLAB_WEBSERVICE_SERVICE_NAME: {'condition': 'service_started'}, }) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks', provider_self_alias=False) | indent(4) }} {% set service_name = GITLAB_GITALY_SERVICE_NAME %} diff --git a/roles/web-app-gitlab/templates/config/smtp_settings.rb.j2 b/roles/web-app-gitlab/templates/config/smtp_settings.rb.j2 index f7c112fc55..4bd2f1188f 100644 --- a/roles/web-app-gitlab/templates/config/smtp_settings.rb.j2 +++ b/roles/web-app-gitlab/templates/config/smtp_settings.rb.j2 @@ -3,10 +3,15 @@ ActionMailer::Base.delivery_method = :smtp ActionMailer::Base.smtp_settings = { address: {{ GITLAB_EMAIL.host | ruby_dq }}, port: {{ GITLAB_EMAIL.port }}, +{% if GITLAB_EMAIL.auth | bool %} user_name: {{ GITLAB_EMAIL.username | ruby_dq }}, password: {{ GITLAB_EMAIL.password | ruby_dq }}, + authentication: :login, +{% endif %} domain: {{ GITLAB_EMAIL.domain | ruby_dq }}, - authentication: {{ ':login' if (GITLAB_EMAIL.auth | bool) else 'nil' }}, +{% if GITLAB_EMAIL_CA_FILE | length > 0 %} + ca_file: {{ GITLAB_EMAIL_CA_FILE | ruby_dq }}, +{% endif %} enable_starttls_auto: {{ GITLAB_EMAIL.start_tls | bool | string | lower }}, - tls: {{ GITLAB_EMAIL.tls | bool | string | lower }} + tls: {{ ((GITLAB_EMAIL.tls | bool) and not (GITLAB_EMAIL.start_tls | bool)) | string | lower }} } diff --git a/roles/web-app-gitlab/templates/playwright.env.j2 b/roles/web-app-gitlab/templates/playwright.env.j2 index 9533f742b4..a4cc1aea35 100644 --- a/roles/web-app-gitlab/templates/playwright.env.j2 +++ b/roles/web-app-gitlab/templates/playwright.env.j2 @@ -1,3 +1,5 @@ +ADMIN_EMAIL={{ lookup('users', 'administrator').email | dotenv_quote }} + ADMIN_PASSWORD={{ lookup('users', 'administrator').password | dotenv_quote }} ADMIN_USERNAME={{ lookup('users', 'administrator').username | dotenv_quote }} @@ -15,7 +17,6 @@ CONFLUENCE_SERVICE_ENABLED={{ lookup('config', application_id, 'services.conflue # nocheck: playwright-service-gate — web-svc-css is injected at the vhost level; assertCspInjections resolves the flag through a loop variable, never a literal gate CSS_SERVICE_ENABLED={{ lookup('config', application_id, 'services.css.enabled') | string | lower }} -# nocheck: playwright-service-gate — templates/config/smtp_settings.rb.j2 wires ActionMailer outbound-only; no persona surface sends mail, real delivery is asserted in roles/web-app-mailu/files/playwright/playwright.spec.js EMAIL_SERVICE_ENABLED={{ lookup('config', application_id, 'services.email.enabled') | string | lower }} JENKINS_SERVICE_ENABLED={{ lookup('config', application_id, 'services.jenkins.enabled') | string | lower }} @@ -53,4 +54,6 @@ SEAWEEDFS_SERVICE_ENABLED={{ lookup('config', application_id, 'services.seaweedf SSO_SERVICE_ENABLED={{ lookup('config', application_id, 'services.sso.enabled') | string | lower }} +WEBMAIL_BASE_URL={{ (lookup('canonical_url', MAIL_PROVIDER, 'webmail') if lookup('config', MAIL_PROVIDER, 'domains.canonical.webmail', '') | length > 0 else '') | dotenv_quote }} + {{ lookup('addon_env_flags', application_id) }} diff --git a/roles/web-app-gitlab/vars/main.yml b/roles/web-app-gitlab/vars/main.yml index 4fef220b53..cf0ecc9e43 100644 --- a/roles/web-app-gitlab/vars/main.yml +++ b/roles/web-app-gitlab/vars/main.yml @@ -19,6 +19,7 @@ GITLAB_SHELL_METRICS_PORT: "{{ lookup('config', application_id, 'services.shell. GITLAB_EMAIL: "{{ lookup('email', application_id) }}" GITLAB_EMAIL_ENABLED: "{{ GITLAB_EMAIL.enabled | bool }}" +GITLAB_EMAIL_CA_FILE: "{{ CA_TRUST.inject_cert_container if lookup('ca_injected', application_id) else '' }}" GITLAB_OIDC_ENABLED: "{{ lookup('config', application_id, 'services.sso.enabled') }}" GITLAB_OIDC_LABEL: "{{ OIDC.BUTTON_TEXT }}" diff --git a/roles/web-app-jitsi/meta/services.yml b/roles/web-app-jitsi/meta/services.yml index af84a95301..4b59b8db4e 100644 --- a/roles/web-app-jitsi/meta/services.yml +++ b/roles/web-app-jitsi/meta/services.yml @@ -31,8 +31,8 @@ prometheus: # nocheck: playwright-service-flag - email not used by Jitsi room flow email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag - css consumer, no Playwright surface (provider: web-svc-css) css: bond: 1 diff --git a/roles/web-app-joomla/meta/services.yml b/roles/web-app-joomla/meta/services.yml index 6526c27d7c..46483ce2ee 100644 --- a/roles/web-app-joomla/meta/services.yml +++ b/roles/web-app-joomla/meta/services.yml @@ -25,8 +25,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — DB engine, covered by role-local integration tests mariadb: bond: 1 @@ -63,6 +63,7 @@ joomla: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta cpus: "0.5" diff --git a/roles/web-app-keycloak/files/playwright/admin-console.js b/roles/web-app-keycloak/files/playwright/admin-console.js new file mode 100644 index 0000000000..581f85f7b0 --- /dev/null +++ b/roles/web-app-keycloak/files/playwright/admin-console.js @@ -0,0 +1,107 @@ +/** + * Keycloak user provisioning through the admin console. + * + * `provisionKeycloakUser(browser, options)` + * Signs the master-realm administrator into the admin console, creates + * `options.user` in `options.realm` when it is missing, and sets + * `options.password` as its permanent password either way, so a user + * left on a reused stack always ends up on the inventory's value. + * + * Args: + * browser: Playwright `Browser`; the flow runs in its own context. + * options.baseUrl: Keycloak base URL (no trailing slash). + * options.realm: realm the user belongs to. + * options.adminUsername / options.adminPassword: master-realm admin. + * options.user: `{ username, firstName, lastName, email }`. + * options.password: password to set on the user. + */ + +const { expect } = require("@playwright/test"); +const { performKeycloakLoginForm, gotoOnion } = require("./personas"); +const { resolveTimeout } = require("./timeouts"); + +async function openUser(page, username) { + const search = page.getByPlaceholder(/search user/i).first(); + await search.waitFor({ state: "visible", timeout: resolveTimeout(60_000) }); + await search.fill(username); + await search.press("Enter"); + const link = page.getByRole("link", { name: username, exact: true }).first(); + const exists = await link + .waitFor({ state: "visible", timeout: resolveTimeout(10_000) }) + .then(() => true) + .catch(() => false); + if (exists) { + await link.click({ timeout: resolveTimeout(30_000) }); + } + return exists; +} + +async function createUser(page, user) { + await page + .getByRole("button", { name: /add user|create new user/i }) + .or(page.getByRole("link", { name: /add user|create new user/i })) + .first() + .click({ timeout: resolveTimeout(30_000) }); + await page.getByLabel(/^username/i).first().fill(user.username); + await page.getByLabel(/^email$/i).first().fill(user.email); + await page.getByLabel(/^first name/i).first().fill(user.firstName); + await page.getByLabel(/^last name/i).first().fill(user.lastName); + await page.getByRole("button", { name: /^create$/i }).click({ timeout: resolveTimeout(30_000) }); + await expect( + page.getByText(/the user has been created/i).first(), + `Keycloak must confirm that ${user.username} was created`, + ).toBeVisible({ timeout: resolveTimeout(60_000) }); +} + +async function setPassword(page, username, password) { + await page.getByRole("tab", { name: /credentials/i }).click({ timeout: resolveTimeout(30_000) }); + await page + .getByRole("button", { name: /^(set|reset) password$/i }) + .first() + .click({ timeout: resolveTimeout(30_000) }); + const dialog = page.getByRole("dialog"); + const passwordFields = dialog.locator("input[type='password']"); + await passwordFields.first().waitFor({ state: "visible", timeout: resolveTimeout(30_000) }); + await passwordFields.nth(0).fill(password); + await passwordFields.nth(1).fill(password); + await dialog + .locator("input[type='checkbox']") + .first() + .setChecked(false, { force: true, timeout: resolveTimeout(30_000) }); + await dialog.getByRole("button", { name: /^save$/i }).click({ timeout: resolveTimeout(30_000) }); + const confirm = page.getByRole("dialog").getByRole("button", { name: /^(save|reset) password$/i }); + const confirmShown = await confirm + .waitFor({ state: "visible", timeout: resolveTimeout(5_000) }) + .then(() => true) + .catch(() => false); + if (confirmShown) { + await confirm.click({ timeout: resolveTimeout(30_000) }); + } + await expect( + page.getByText(/password has been (set|reset) successfully/i).first(), + `Keycloak must confirm the new password of ${username}`, + ).toBeVisible({ timeout: resolveTimeout(60_000) }); +} + +async function provisionKeycloakUser(browser, options) { + const { baseUrl, realm, adminUsername, adminPassword, user, password } = options; + const context = await browser.newContext({ ignoreHTTPSErrors: true }); + const page = await context.newPage(); + try { + await gotoOnion(page, `${baseUrl}/admin/master/console/#/${realm}/users`); + await performKeycloakLoginForm(page, adminUsername, adminPassword); + if (!(await openUser(page, user.username))) { + await createUser(page, user); + } + await setPassword(page, user.username, password); + await page + .goto(`${baseUrl}/realms/master/protocol/openid-connect/logout`, { waitUntil: "commit" }) + .catch(() => {}); + } finally { + await context.close(); + } +} + +module.exports = { + provisionKeycloakUser, +}; diff --git a/roles/web-app-keycloak/files/playwright/playwright.spec.js b/roles/web-app-keycloak/files/playwright/playwright.spec.js index 2b93f74a2e..46b56b5c07 100644 --- a/roles/web-app-keycloak/files/playwright/playwright.spec.js +++ b/roles/web-app-keycloak/files/playwright/playwright.spec.js @@ -2,7 +2,8 @@ const { test, expect } = require("@playwright/test"); const { resolveTimeout } = require("./timeouts"); const { skipUnlessServiceEnabled } = require("./service-gating"); -const { assertCspMetaParity, assertCspResponseHeader, decodeDotenvQuotedValue, expectNoCspViolations, gotoOnion, installCspViolationObserver, normalizeBaseUrl, runAdminFlow, runBiberFlow, runGuestFlow, safeSkipUnlessEnabled } = require("./personas"); +const { MAPACHE, assertCspMetaParity, assertCspResponseHeader, decodeDotenvQuotedValue, expectNoCspViolations, gotoOnion, installCspViolationObserver, normalizeBaseUrl, runAdminFlow, runBiberFlow, runGuestFlow, safeSkipUnlessEnabled } = require("./personas"); +const { provisionKeycloakUser } = require("./admin-console"); test.use({ ignoreHTTPSErrors: true }); // ----------------------------------------------------------------------------- @@ -82,6 +83,8 @@ const adminUsername = decodeDotenvQuotedValue(process.env.ADMIN_USERNAME); const adminPassword = decodeDotenvQuotedValue(process.env.ADMIN_PASSWORD); const biberUsername = decodeDotenvQuotedValue(process.env.BIBER_USERNAME); const biberPassword = decodeDotenvQuotedValue(process.env.BIBER_PASSWORD); +const domainPrimary = decodeDotenvQuotedValue(process.env.DOMAIN_PRIMARY); +const mapachePassword = decodeDotenvQuotedValue(process.env.MAPACHE_PASSWORD); const canonicalDomain = decodeDotenvQuotedValue(process.env.CANONICAL_DOMAIN); test.beforeEach(async ({ page }) => { @@ -198,6 +201,48 @@ test("normal-realm administrator logs in through account interface and logs out" await expectNoCspViolations(page, diagnostics, "keycloak normal-realm account (administrator)"); }); +test("super administrator onboards mapache in the admin console and mapache signs in", async ({ page, browser }) => { + const diagnostics = attachDiagnostics(page); + + expect(domainPrimary, "DOMAIN_PRIMARY must be set in the Playwright env file").toBeTruthy(); + expect(mapachePassword, "MAPACHE_PASSWORD must be set in the Playwright env file").toBeTruthy(); + + const mapacheUsername = MAPACHE.username; + await provisionKeycloakUser(browser, { + baseUrl: appBaseUrl, + realm: realmName, + adminUsername: superAdminUsername, + adminPassword: superAdminPassword, + user: { ...MAPACHE, email: `${MAPACHE.username}@${domainPrimary}` }, + password: mapachePassword, + }); + + const accountUrl = `${appBaseUrl}/realms/${realmName}/account/`; + const response = await gotoOnion(page, accountUrl); + expect(response, "Expected normal-realm account page response").toBeTruthy(); + expect(response.status(), "Expected normal-realm account page response to be successful").toBeLessThan(400); + + const signInButton = page.locator("a, button").filter({ hasText: /sign\s*in|log\s*in|anmelden/i }).first(); + if ((await signInButton.count().catch(() => 0)) > 0) { + await signInButton.click({ timeout: resolveTimeout(30_000) }); + } + + await fillKeycloakLoginForm(page, mapacheUsername, mapachePassword); + + await expect + .poll(() => page.url(), { + timeout: resolveTimeout(60_000), + message: "Expected mapache login to reach the account interface" + }) + .toContain("/account"); + + await expect(page.locator("body")).toContainText(/personal\s*info|account|profile|signing\s*in/i, { timeout: resolveTimeout(60_000) }); + + await keycloakSignOutFromAccountConsole(page); + + await expectNoCspViolations(page, diagnostics, "keycloak normal-realm account (mapache)"); +}); + test("normal-realm biber logs in through account interface and logs out", async ({ page }) => { safeSkipUnlessEnabled("ldap"); const diagnostics = attachDiagnostics(page); diff --git a/roles/web-app-keycloak/filter_plugins/oidc_clients.py b/roles/web-app-keycloak/filter_plugins/oidc_clients.py new file mode 100644 index 0000000000..4696969017 --- /dev/null +++ b/roles/web-app-keycloak/filter_plugins/oidc_clients.py @@ -0,0 +1,79 @@ +"""Keycloak clients that deployed applications declare for themselves. + +An application that needs more than the one shared client declares them in its +own ``meta/services.yml``:: + + services: + sso: + oidc: + clients: + webui: myapp-admin + webmail: myapp-webmail + +Keycloak converges whatever it finds, so no application id is named on this +side. The realm's built-in clients (``account``, ``broker``, ...) are not +declared by anybody and therefore never touched. +""" + +from __future__ import annotations + +from typing import Any + + +def _declared(app: Any) -> dict[str, Any]: + if not isinstance(app, dict): + return {} + node: Any = app + for key in ("services", "sso", "oidc", "clients"): + if not isinstance(node, dict): + return {} + node = node.get(key) + return node if isinstance(node, dict) else {} + + +def kc_declared_client_apps(applications: Any, app_ids: Any = None) -> list[str]: + """Application ids (of the given ones) that declare their own clients. + + Each of these ships ``templates/keycloak/clients.json.j2`` — the realm + import includes that fragment per app. + + Args: + applications: the ``lookup('applications')`` map, application id -> config. + app_ids: restrict to these ids; falsy means every application in the map. + + Returns: + Sorted application ids with a non-empty ``services.sso.oidc.clients``. + """ + if not isinstance(applications, dict): + return [] + wanted = list(app_ids) if app_ids else list(applications) + return sorted(a for a in wanted if _declared(applications.get(a))) + + +def kc_declared_client_ids(applications: Any, app_ids: Any = None) -> list[str]: + """Every client id declared by the given applications, sorted and deduped. + + Args: + applications: the ``lookup('applications')`` map, application id -> config. + app_ids: restrict to these ids; falsy means every application in the map. + + Returns: + Sorted unique client ids. Empty when nothing declares any. + """ + if not isinstance(applications, dict): + return [] + wanted = list(app_ids) if app_ids else list(applications) + found: set[str] = set() + for app_id in wanted: + for value in _declared(applications.get(app_id)).values(): + if isinstance(value, str) and value.strip(): + found.add(value.strip()) + return sorted(found) + + +class FilterModule: + def filters(self): + return { + "kc_declared_client_apps": kc_declared_client_apps, + "kc_declared_client_ids": kc_declared_client_ids, + } diff --git a/roles/web-app-keycloak/meta/services.yml b/roles/web-app-keycloak/meta/services.yml index 658c286a58..7e41c7df52 100644 --- a/roles/web-app-keycloak/meta/services.yml +++ b/roles/web-app-keycloak/meta/services.yml @@ -26,8 +26,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" javascript: bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" @@ -51,7 +51,7 @@ keycloak: shared: true provides: sso image: quay.io/keycloak/keycloak - version: "26.7.4" + version: "26.8.0" name: keycloak min_storage: 468MB cpus: "2.0" @@ -81,13 +81,14 @@ keycloak: service_key: keycloak run_after: - web-app-matomo + - web-app-stalwart - web-app-mailu lifecycle: beta # nocheck: dynamic-flag — per-consumer fan-out # nocheck: playwright-service-flag — sidecar shape, no probe-able surface flag sso_proxy: image: quay.io/oauth2-proxy/oauth2-proxy - version: "v7.15.4" + version: "v7.15.5" min_storage: 220MB configuration_file: oauth2-proxy-keycloak.cfg ports: diff --git a/roles/web-app-keycloak/tasks/03_manager_ops/update_routines.yml b/roles/web-app-keycloak/tasks/03_manager_ops/update_routines.yml index b8f5576134..2eb7d792cd 100644 --- a/roles/web-app-keycloak/tasks/03_manager_ops/update_routines.yml +++ b/roles/web-app-keycloak/tasks/03_manager_ops/update_routines.yml @@ -50,10 +50,19 @@ include_tasks: "{{ role_path }}/tasks/update/08_users/email_verified.yml" when: KEYCLOAK_USER_EMAILS_VERIFIED | bool +- name: "📂 Ensure application-declared clients (existing realms)" + include_tasks: "{{ role_path }}/tasks/update/10_declared_client.yml" + vars: + kc_declared_client_id: "{{ declared_client }}" + loop: "{{ KEYCLOAK_DECLARED_CLIENT_IDS }}" + loop_control: + loop_var: declared_client + - name: "⏳ Reap the remaining Keycloak update jobs" ansible.builtin.include_tasks: utils/async/reap.yml vars: async_reap_jobs: >- - {{ kc_rbac_default.results | default([kc_rbac_default] if kc_rbac_default is defined else []) }} + {{ (kc_rbac_default.results | default([kc_rbac_default] if kc_rbac_default is defined else [])) + + (kc_declared_client_jobs | default([])) }} async_reap_label: "Keycloak realm updates" async_reap_tolerate: [] diff --git a/roles/web-app-keycloak/tasks/update/10_declared_client.yml b/roles/web-app-keycloak/tasks/update/10_declared_client.yml new file mode 100644 index 0000000000..06a5934de8 --- /dev/null +++ b/roles/web-app-keycloak/tasks/update/10_declared_client.yml @@ -0,0 +1,29 @@ +--- +# Realm import only runs for a MISSING realm; this kcadm create-or-update brings +# application-declared clients to existing realms too. Desired state is the +# import's own client object, so the realm template stays the one SPOT. +- name: "✅ Ensure the '{{ kc_declared_client_id }}' client" + vars: + kc_desired: >- + {{ + KEYCLOAK_DICTIONARY_REALM.clients + | selectattr('clientId', 'equalto', kc_declared_client_id) + | list | first + }} + keycloak_kcadm_update: + object_kind: "client" + lookup_value: "{{ kc_declared_client_id }}" + desired: "{{ kc_desired }}" + kcadm_exec: "{{ KEYCLOAK_EXEC_KCADM }}" + realm: "{{ KEYCLOAK_REALM }}" + assert_mode: "{{ MODE_ASSERT }}" + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + register: kc_declared_client + # rationale: async; nothing reads the client back in this chain, reaped at the end of update_routines.yml + async: "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" + poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" + +# rationale: looped include — each iteration overwrites the register, so the job handle is banked here for the reap. +- name: "🧾 Queue the '{{ kc_declared_client_id }}' client job for the reap" + ansible.builtin.set_fact: + kc_declared_client_jobs: "{{ (kc_declared_client_jobs | default([])) + [kc_declared_client] }}" diff --git a/roles/web-app-keycloak/templates/import/clients/default.json.j2 b/roles/web-app-keycloak/templates/import/clients/default.json.j2 index a8add45b22..0372afd7b0 100644 --- a/roles/web-app-keycloak/templates/import/clients/default.json.j2 +++ b/roles/web-app-keycloak/templates/import/clients/default.json.j2 @@ -47,6 +47,7 @@ "profile", "basic", "email", + "openid", "{{ KEYCLOAK_RBAC_GROUP_CLAIM }}" ], "optionalClientScopes": [ diff --git a/roles/web-app-keycloak/templates/import/realm.json.j2 b/roles/web-app-keycloak/templates/import/realm.json.j2 index bbc2389dcd..f8b3e85afb 100644 --- a/roles/web-app-keycloak/templates/import/realm.json.j2 +++ b/roles/web-app-keycloak/templates/import/realm.json.j2 @@ -791,9 +791,21 @@ ] }, {% include "clients/default.json.j2" %} +{% for kc_client_app in KEYCLOAK_DECLARED_CLIENT_APPS %} + ,{% include 'roles/' ~ kc_client_app ~ '/templates/keycloak/clients.json.j2' %} +{% endfor %} {% include "clients/saml.json.j2" %} ], "clientScopes": [ + { + "name": "openid", + "description": "Marker scope so 'scope=openid' requests are valid and echoed back; some OIDC clients (e.g. Mattermost) require it. Carries no mappers.", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + } + }, { "name": "role_list", "description": "SAML role list", diff --git a/roles/web-app-keycloak/templates/playwright.env.j2 b/roles/web-app-keycloak/templates/playwright.env.j2 index 6aa3690410..a0595111b4 100644 --- a/roles/web-app-keycloak/templates/playwright.env.j2 +++ b/roles/web-app-keycloak/templates/playwright.env.j2 @@ -10,6 +10,8 @@ BIBER_USERNAME={{ lookup('users', 'biber').username | dotenv_quote }} CANONICAL_DOMAIN={{ lookup('domain', application_id) | dotenv_quote }} +DOMAIN_PRIMARY={{ DOMAIN_PRIMARY | dotenv_quote }} + # nocheck: playwright-service-gate — covered by sys-svc-mail integration tests, not the persona surface EMAIL_SERVICE_ENABLED={{ lookup('config', application_id, 'services.email.enabled') | string | lower }} @@ -26,6 +28,8 @@ LOGOUT_PANEL_DE_JSON={{ lookup('i18n_catalogue', 'logout', 'roles/web-app-keyclo # nocheck: playwright-service-gate — auth-provider role; generic persona scenarios are exempt (covered by bespoke realm-account / admin-console tests above) LOGOUT_SERVICE_ENABLED={{ lookup('config', application_id, 'services.logout.enabled') | string | lower }} +MAPACHE_PASSWORD={{ PLAYWRIGHT_MAPACHE_PASSWORD | dotenv_quote }} + MATOMO_BASE_URL={{ lookup('tls', 'web-app-matomo', 'url.base') | dotenv_quote }} # nocheck: playwright-service-gate — auth-provider role; generic persona scenarios are exempt diff --git a/roles/web-app-keycloak/vars/main.yml b/roles/web-app-keycloak/vars/main.yml index 9c8849588c..dd43b34b61 100644 --- a/roles/web-app-keycloak/vars/main.yml +++ b/roles/web-app-keycloak/vars/main.yml @@ -121,6 +121,9 @@ KEYCLOAK_USER_EMAILS_VERIFIED: "{{ lookup('config', application_id, 'services.ke KEYCLOAK_MOODLE_ENABLED: "{{ 'web-app-moodle' in group_names }}" +KEYCLOAK_DECLARED_CLIENT_APPS: "{{ lookup('applications') | kc_declared_client_apps(lookup('deployment').groups) }}" +KEYCLOAK_DECLARED_CLIENT_IDS: "{{ lookup('applications') | kc_declared_client_ids(lookup('deployment').groups) }}" + KEYCLOAK_RECAPTCHA_ENABLED: "{{ lookup('config', application_id, 'services.recaptcha.enabled') }}" KEYCLOAK_RECAPTCHA_KEY: "{{ lookup('config', application_id, 'secrets.credentials.recaptcha.key') }}" KEYCLOAK_RECAPTCHA_SECRET: "{{ lookup('config', application_id, 'secrets.credentials.recaptcha.secret') }}" diff --git a/roles/web-app-kix/meta/services.yml b/roles/web-app-kix/meta/services.yml index fcb66ca59f..c5da3a89d7 100644 --- a/roles/web-app-kix/meta/services.yml +++ b/roles/web-app-kix/meta/services.yml @@ -14,8 +14,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" ldap: bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" diff --git a/roles/web-app-kix/templates/compose.yml.j2 b/roles/web-app-kix/templates/compose.yml.j2 index f09e1e0461..11ad449fc5 100644 --- a/roles/web-app-kix/templates/compose.yml.j2 +++ b/roles/web-app-kix/templates/compose.yml.j2 @@ -10,6 +10,7 @@ {% include 'roles/sys-svc-container/templates/base.yml.j2' %} {{ lookup('container_depends_on', application_id) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} # nocheck: healthcheck-timing-override the image's own HEALTHCHECK supplies test, interval, timeout and retries; only the start window is corrected here healthcheck: @@ -34,6 +35,7 @@ - NOTIFICATION_URL=http://{{ KIX_FRONTEND_SERVICE }}:{{ KIX_FRONTEND_INTERNAL_PORT }}/notifications - BACKEND_API_URL=http://{{ KIX_BACKEND_SERVICE }}:{{ KIX_BACKEND_INTERNAL_PORT }}/api/v1 {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} # nocheck: healthcheck-timing-override the image's own HEALTHCHECK supplies test, interval, timeout and retries; only the start window is corrected here healthcheck: @@ -48,6 +50,7 @@ {{ lookup('depends_on', {KIX_BACKEND_SERVICE: 'service_healthy', KIX_FRONTEND_SERVICE: 'service_healthy'}) }} {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('container_ports', ['kix', 'http', DOCKER_BIND_HOST]) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('compose_volumes') }} diff --git a/roles/web-app-listmonk/meta/services.yml b/roles/web-app-listmonk/meta/services.yml index 1f9ddd6cf7..406095f0f0 100644 --- a/roles/web-app-listmonk/meta/services.yml +++ b/roles/web-app-listmonk/meta/services.yml @@ -23,8 +23,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" postgres: bond: 1 enabled: true # nocheck: dynamic-flag @@ -63,6 +63,7 @@ listmonk: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta # nocheck: playwright-service-flag — object store, no Playwright surface diff --git a/roles/web-app-listmonk/templates/compose.yml.j2 b/roles/web-app-listmonk/templates/compose.yml.j2 index ada0d37195..6b4f2ddb11 100644 --- a/roles/web-app-listmonk/templates/compose.yml.j2 +++ b/roles/web-app-listmonk/templates/compose.yml.j2 @@ -7,6 +7,7 @@ command: ["sh", "-c", "./listmonk --install --idempotent --yes && ./listmonk --upgrade --yes && ./listmonk"] {{ lookup('container_ports', ['listmonk', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} diff --git a/roles/web-app-listmonk/vars/main.yml b/roles/web-app-listmonk/vars/main.yml index c8887f7d35..6497dc9956 100644 --- a/roles/web-app-listmonk/vars/main.yml +++ b/roles/web-app-listmonk/vars/main.yml @@ -97,7 +97,7 @@ LISTMONK_SETTINGS: "uuid": "471fd0e9-8c33-4e4a-9183-c4679699faca", "enabled": true, "username": lookup('users', LISTMONK_USER_BOUNCE).email, - "password": lookup('users', LISTMONK_USER_BOUNCE).get('tokens', {}).get('web-app-mailu', ''), + "password": lookup('users', LISTMONK_USER_BOUNCE).get('tokens', {}).get(MAIL_PROVIDER, ''), "return_path": lookup('users', LISTMONK_USER_BOUNCE).email, "tls_enabled": LISTMONK_EMAIL.tls, "auth_protocol": "userpass", diff --git a/roles/web-app-magento/meta/services.yml b/roles/web-app-magento/meta/services.yml index ad81b2dcdf..b57506fa6f 100644 --- a/roles/web-app-magento/meta/services.yml +++ b/roles/web-app-magento/meta/services.yml @@ -25,8 +25,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" php: image: markoshust/magento-php version: 8.4-fpm # nocheck: docker-version — magento/product-community-edition 2.4.8 requires php ~8.2.0||~8.3.0||~8.4.0; dc08bc621 already pinned this down from 8.5 and the image bot reverted it in 1e4f82128, so composer install fails again diff --git a/roles/web-app-magento/templates/compose.yml.j2 b/roles/web-app-magento/templates/compose.yml.j2 index dc73442b8e..5f90996d7a 100644 --- a/roles/web-app-magento/templates/compose.yml.j2 +++ b/roles/web-app-magento/templates/compose.yml.j2 @@ -16,6 +16,7 @@ retries: 5 start_period: 10m start_interval: 5s + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'php' %} @@ -27,6 +28,7 @@ {{ lookup('container_depends_on', application_id, extra={ 'search': {'condition': 'service_started'}, }) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% set service_name = 'search' %} @@ -39,6 +41,7 @@ - plugins.security.disabled=true - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m {{ lookup('container_healthcheck', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('compose_volumes') }} diff --git a/roles/web-app-mailu/README.md b/roles/web-app-mailu/README.md index 453d11c21f..4568f8c4cd 100644 --- a/roles/web-app-mailu/README.md +++ b/roles/web-app-mailu/README.md @@ -1,5 +1,12 @@ # Mailu +> **⚠️ Deprecated.** Mailu is superseded by [`web-app-stalwart`](../web-app-stalwart/), +> the default email provider. The role keeps working until it is removed, but new +> deployments MUST NOT adopt it and existing ones MUST migrate. Follow +> [Migration from Mailu](../web-app-stalwart/README.md#migration-from-mailu) to move +> the mailboxes over. Until then, set `MAIL_PROVIDER: web-app-mailu` to keep routing +> the mail abstraction (`plugins/lookup/email.py`, `sys-svc-mail`) to Mailu. + ## Description Revolutionize your email communications with Mailu, a secure and flexible mail server solution that integrates comprehensive features like robust SMTP/IMAP support, advanced spam filtering, DKIM signing, and seamless webmail access. With its modern design and performance-oriented architecture, Mailu empowers you to manage digital correspondence with efficiency and reliability. diff --git a/roles/web-app-mailu/meta/domains.yml b/roles/web-app-mailu/meta/domains.yml index e63a9e737a..15e9545aaf 100644 --- a/roles/web-app-mailu/meta/domains.yml +++ b/roles/web-app-mailu/meta/domains.yml @@ -1,4 +1,6 @@ --- +# Owns `mail.` only when selected as the active provider; otherwise it parks on a +# legacy domain so it never collides with the default (web-app-stalwart). canonical: - - mail.{{ DOMAIN_PRIMARY }} + - "{{ ('mail.' ~ DOMAIN_PRIMARY) if MAIL_PROVIDER == 'web-app-mailu' else ('legacy-mail.' ~ DOMAIN_PRIMARY) }}" aliases: [] diff --git a/roles/web-app-mailu/meta/server.yml b/roles/web-app-mailu/meta/server.yml index f5047bd3f2..2d8044978c 100644 --- a/roles/web-app-mailu/meta/server.yml +++ b/roles/web-app-mailu/meta/server.yml @@ -1,5 +1,6 @@ --- status_codes: default: + - 302 - 301 - 200 diff --git a/roles/web-app-mailu/meta/services.yml b/roles/web-app-mailu/meta/services.yml index 5bb86d4361..dd0f5ed025 100644 --- a/roles/web-app-mailu/meta/services.yml +++ b/roles/web-app-mailu/meta/services.yml @@ -20,6 +20,11 @@ matomo: bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" +# nocheck: dynamic-flag +email: + bond: 1 + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — self-provider mailu: modes: @@ -32,7 +37,8 @@ mailu: placement: manager enabled: true # nocheck: dynamic-flag shared: true - provides: email + covers: + - email # nocheck: unwatched-version Reason: Mailu ships its images under the release-series tag while its git tags are patch level (2024.06.58), so the series is the pin and moves only on a deliberate migration version: "2024.06" name: mailu @@ -73,7 +79,8 @@ mailu: service_key: front run_after: - web-app-matomo - lifecycle: beta + - web-app-stalwart + lifecycle: deprecated cpus: "1.0" mem_reservation: 512m mem_limit: 2g diff --git a/roles/web-app-mailu/meta/variants.yml b/roles/web-app-mailu/meta/variants.yml index 7b87878469..c2c5a6146e 100644 --- a/roles/web-app-mailu/meta/variants.yml +++ b/roles/web-app-mailu/meta/variants.yml @@ -14,6 +14,9 @@ dashboard: enabled: true shared: true + email: + enabled: true + shared: true logout: enabled: true shared: true @@ -42,6 +45,9 @@ dashboard: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true + email: + enabled: false + shared: false # nocheck: variants-static-override backup shared stays literally true logout: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true diff --git a/roles/web-app-mailu/tasks/00_core.yml b/roles/web-app-mailu/tasks/00_core.yml index 7af224ef91..76d2104c99 100644 --- a/roles/web-app-mailu/tasks/00_core.yml +++ b/roles/web-app-mailu/tasks/00_core.yml @@ -49,7 +49,7 @@ compose_handlers_flush: true webserver_vhost_flavour: "basic" webserver_extra_configuration: "client_max_body_size 31M;" - domain: "{{ MAILU_HOSTNAME }}" + domain: "{{ MAILU_CANONICAL_HOST }}" - name: "👔 Mailu manager-side operations (file deploys + container exec)" include_tasks: 02_manager_ops.yml diff --git a/roles/web-app-mailu/tasks/02_manager_ops/post_bootstrap_ops.yml b/roles/web-app-mailu/tasks/02_manager_ops/post_bootstrap_ops.yml index 658b4be726..e29ee062de 100644 --- a/roles/web-app-mailu/tasks/02_manager_ops/post_bootstrap_ops.yml +++ b/roles/web-app-mailu/tasks/02_manager_ops/post_bootstrap_ops.yml @@ -37,6 +37,7 @@ - name: "📂 Include Cert deploy service for '{{ role_name }}'" include_role: name: sys-ctl-mtn-cert-deploy + when: (lookup('tls', application_id, 'enabled') | bool) or (MAILU_ACTIVE_PROVIDER | bool) - name: ⏳ Wait for Mailu antispam include_tasks: ../06_wait_for_antispam.yml @@ -46,3 +47,7 @@ - name: 🔧 Set Mailu DNS records include_tasks: ../08_dns-records.yml + +- name: "🔐 Apply Mailu OIDC provider config" + include_tasks: ../09_oidc_apply.yml + when: MAILU_OIDC_ENABLED | bool diff --git a/roles/web-app-mailu/tasks/09_oidc_apply.yml b/roles/web-app-mailu/tasks/09_oidc_apply.yml new file mode 100644 index 0000000000..3fa0531552 --- /dev/null +++ b/roles/web-app-mailu/tasks/09_oidc_apply.yml @@ -0,0 +1,17 @@ +--- +# Always defer the OIDC refresh to the service-loader post-load flush: mid-play +# the issuer probe can hit a stale Keycloak that a later round step replaces. +# The flush runs after the loader finished, i.e. against the final Keycloak +# (10_oidc_refresh still waits for the issuer and fails loudly if absent). + +- name: "📥 Defer the OIDC refresh to the service-loader post-load flush" + ansible.builtin.set_fact: + mailu_oidc_restart_pending: true + +- name: "📥 Queue the deferred refresh" + include_role: + name: sys-service-loader + tasks_from: list_or_shoot.yml + vars: + list_or_shoot_roles: + - web-app-mailu diff --git a/roles/web-app-mailu/tasks/10_oidc_refresh.yml b/roles/web-app-mailu/tasks/10_oidc_refresh.yml new file mode 100644 index 0000000000..7ef8b52193 --- /dev/null +++ b/roles/web-app-mailu/tasks/10_oidc_refresh.yml @@ -0,0 +1,29 @@ +--- +# Restart admin only while the issuer answers: heviat-Mailu fetches the OIDC +# provider config on the first /sso request and caches a failure until restart. + +- name: "⏳ Wait for the OIDC issuer discovery endpoint" + ansible.builtin.shell: >- + container exec -i {{ MAILU_ADMIN_CONTAINER_ADDRESS }} + {{ 30 | curl }} -ksS -o /dev/null -w "%{http_code}" + --proxy {{ MAILU_OIDC_SOCKS_PROXY | quote }} {{ OIDC.CLIENT.DISCOVERY_DOCUMENT }} + register: mailu_oidc_issuer_probe + changed_when: false + retries: 30 + delay: 5 + until: mailu_oidc_issuer_probe.stdout == '200' + +- name: "🔄 Restart Mailu admin so the OIDC provider config is re-fetched (compose)" + ansible.builtin.command: "container restart {{ MAILU_ADMIN_CONTAINER_ADDRESS }}" + changed_when: true + when: DEPLOYMENT_MODE == 'compose' + +- name: "🔄 Restart Mailu admin so the OIDC provider config is re-fetched (swarm)" + ansible.builtin.command: "container service update --force {{ lookup('container_service', application_id, 'admin') }}" + changed_when: true + timeout: "{{ lookup('timeout', 600) | int }}" + when: DEPLOYMENT_MODE == 'swarm' + +- name: "🧹 Clear the deferred OIDC refresh flag" + ansible.builtin.set_fact: + mailu_oidc_restart_pending: false diff --git a/roles/web-app-mailu/tasks/main.yml b/roles/web-app-mailu/tasks/main.yml index 5fb8d95cfe..94b30f055c 100644 --- a/roles/web-app-mailu/tasks/main.yml +++ b/roles/web-app-mailu/tasks/main.yml @@ -3,3 +3,11 @@ when: run_once_web_app_mailu is not defined vars: application_id: "web-app-mailu" + +- include_tasks: 10_oidc_refresh.yml + when: + - run_once_web_app_mailu is defined + - mailu_oidc_restart_pending | default(false) | bool + - not (service_loader_preloading | default(false) | bool) + vars: + application_id: "web-app-mailu" diff --git a/roles/web-app-mailu/templates/compose.yml.j2 b/roles/web-app-mailu/templates/compose.yml.j2 index 97e68840a0..e197cdb575 100644 --- a/roles/web-app-mailu/templates/compose.yml.j2 +++ b/roles/web-app-mailu/templates/compose.yml.j2 @@ -5,7 +5,11 @@ {{ lookup('compose_only', 'container_name', MAILU_FRONT_CONTAINER_ADDRESS) }} {{ lookup('container_image', application_id, 'front', image=MAILU_DOCKER_FLAVOR ~ '/nginx', version=MAILU_VERSION) }} {% include 'roles/sys-svc-container/templates/base.yml.j2' %} + {% if MAILU_ACTIVE_PROVIDER | bool %} {{ lookup('container_ports_mailu') | indent(4) }} + {% else %} + {{ lookup('container_ports', ['mailu', 'http', DOCKER_BIND_HOST]) | indent(4) }} + {% endif %} {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} {{ lookup('container_networks') | indent(4) }} diff --git a/roles/web-app-mailu/templates/env.j2 b/roles/web-app-mailu/templates/env.j2 index 1b3edd0b19..469b70fa0d 100644 --- a/roles/web-app-mailu/templates/env.j2 +++ b/roles/web-app-mailu/templates/env.j2 @@ -4,9 +4,9 @@ LD_PRELOAD= SECRET_KEY={{ MAILU_SECRET_KEY }} SUBNET={{ MAILU_SUBNET }} DOMAIN={{ MAILU_DOMAIN }} -HOSTNAMES={{ MAILU_HOSTNAME }} +HOSTNAMES={{ MAILU_CANONICAL_HOST }} POSTMASTER=admin -TLS_FLAVOR={{ MAILU_EMAIL.tls | ternary('mail','notls') }} +TLS_FLAVOR={{ MAILU_TLS_FLAVOR }} PORTS={{ MAILU_PORTS.values() | map('int') | unique | sort | join(',') }} AUTH_RATELIMIT=10/minute;1000/hour MESSAGE_RATELIMIT_EXEMPTION={{ lookup('users', 'no-reply').email | lower }} diff --git a/roles/web-app-mailu/templates/playwright.env.j2 b/roles/web-app-mailu/templates/playwright.env.j2 index 8e3fa3556d..8ad0a0770b 100644 --- a/roles/web-app-mailu/templates/playwright.env.j2 +++ b/roles/web-app-mailu/templates/playwright.env.j2 @@ -14,6 +14,9 @@ BIBER_USERNAME={{ lookup('users', 'biber').username | dotenv_quote }} CANONICAL_DOMAIN={{ lookup('domain', application_id) | dotenv_quote }} +# nocheck: playwright-service-gate — platform mail provider presence, no Mailu spec surface +EMAIL_SERVICE_ENABLED={{ lookup('config', application_id, 'services.email.enabled') | string | lower }} + LOGOUT_SERVICE_ENABLED={{ lookup('config', application_id, 'services.logout.enabled') | string | lower }} MAILU_BASE_URL={{ lookup('tls', 'web-app-mailu', 'url.base') | dotenv_quote }} diff --git a/roles/web-app-mailu/vars/main.yml b/roles/web-app-mailu/vars/main.yml index 8b3f5be2c7..67132cbd2e 100644 --- a/roles/web-app-mailu/vars/main.yml +++ b/roles/web-app-mailu/vars/main.yml @@ -12,6 +12,9 @@ MAILU_DOMAIN: "{{ lookup('config', application_id, 'services.mailu.domain') }}" MAILU_DOMAIN_DNS_ZONE: "{{ MAILU_DOMAIN | to_zone }}" MAILU_EMAIL: "{{ lookup('email', application_id) }}" MAILU_HOSTNAME: "{{ MAILU_EMAIL.host }}" +MAILU_CANONICAL_HOST: "{{ lookup('domain', application_id) }}" +MAILU_ACTIVE_PROVIDER: "{{ lookup('mail_provider') == application_id }}" +MAILU_TLS_FLAVOR: "{{ (MAILU_EMAIL.tls and MAILU_ACTIVE_PROVIDER | bool) | ternary('mail', 'notls') }}" MAILU_HOSTNAME_DNS_ZONE: "{{ MAILU_HOSTNAME | to_zone }}" MAILU_VERSION: "{{ lookup('config', application_id, 'services.mailu.version') }}" MAILU_PORTS: "{{ lookup('config', application_id, 'services.mailu.ports.internal') }}" @@ -36,6 +39,8 @@ MAILU_SUBNET: "{{ lookup('config', 'web-app-mailu', 'networks.local.subnet') }}" MAILU_SECRET_KEY: "{{ lookup('config', application_id, 'secrets.credentials.secret_key') }}" MAILU_API_TOKEN: "{{ lookup('config', application_id, 'secrets.credentials.api_token') }}" MAILU_OIDC_HOST: "{{ lookup('tls', 'web-app-keycloak', 'domain') }}" +MAILU_OIDC_ONION_ISSUER: "{{ MAILU_OIDC_HOST.endswith('.onion') }}" +MAILU_OIDC_SOCKS_PROXY: "{{ lookup('tor_socks_proxy') if MAILU_OIDC_ONION_ISSUER | bool else '' }}" MAILU_OIDC_ENABLED: "{{ lookup('config', application_id, 'services.sso.enabled') }}" MAILU_OIDC_EMAIL_BY_USERNAME_ENABLED: "{{ lookup('config', application_id, 'services.sso.oidc.email_by_username') }}" MAILU_OIDC_ENABLE_USER_CREATION: "{{ lookup('config', application_id, 'services.sso.oidc.enable_user_creation') | string | capitalize }}" diff --git a/roles/web-app-mastodon/meta/services.yml b/roles/web-app-mastodon/meta/services.yml index bb753b42b0..8059c49612 100644 --- a/roles/web-app-mastodon/meta/services.yml +++ b/roles/web-app-mastodon/meta/services.yml @@ -18,8 +18,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: shared # nocheck: dynamic-flag redis: diff --git a/roles/web-app-mastodon/templates/compose.yml.j2 b/roles/web-app-mastodon/templates/compose.yml.j2 index d52afe0435..960180318e 100644 --- a/roles/web-app-mastodon/templates/compose.yml.j2 +++ b/roles/web-app-mastodon/templates/compose.yml.j2 @@ -24,6 +24,7 @@ {% if with_volumes %} {{ lookup('container_volumes', service_name) | indent(4) }} {% endif %} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% endmacro -%} diff --git a/roles/web-app-matrix/meta/services.yml b/roles/web-app-matrix/meta/services.yml index ed1d3b7e92..e46c035c25 100644 --- a/roles/web-app-matrix/meta/services.yml +++ b/roles/web-app-matrix/meta/services.yml @@ -28,8 +28,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" gitlab: bond: 0.5 enabled: "{{ 'web-app-gitlab' in group_names }}" @@ -173,6 +173,7 @@ matrix: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta min_storage: "1GB" diff --git a/roles/web-app-mattermost/meta/services.yml b/roles/web-app-mattermost/meta/services.yml index 8fbbbac703..7958f80035 100644 --- a/roles/web-app-mattermost/meta/services.yml +++ b/roles/web-app-mattermost/meta/services.yml @@ -21,8 +21,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — DB engine, covered by role-local integration tests postgres: bond: 1 @@ -66,6 +66,7 @@ mattermost: http: 8065 run_after: - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta # nocheck: dynamic-flag diff --git a/roles/web-app-mattermost/tasks/02_keycloak.yml b/roles/web-app-mattermost/tasks/02_keycloak.yml index 5bb86d5921..e95d82fbcd 100644 --- a/roles/web-app-mattermost/tasks/02_keycloak.yml +++ b/roles/web-app-mattermost/tasks/02_keycloak.yml @@ -88,7 +88,6 @@ delegate_to: "{{ MATTERMOST_KC_HOST_NODE }}" register: mm_kc_all_scopes changed_when: false - when: mm_kc_create_openid_scope is changed - name: "📋 Mattermost | Keycloak — list existing default client scopes" ansible.builtin.shell: >- # nocheck: container-exec-resolver MATTERMOST_KC_LOCAL_CID resolved at delegate_to runtime; container_address would return a manager-only subshell diff --git a/roles/web-app-mattermost/templates/compose.yml.j2 b/roles/web-app-mattermost/templates/compose.yml.j2 index e340a31716..3e3249d429 100644 --- a/roles/web-app-mattermost/templates/compose.yml.j2 +++ b/roles/web-app-mattermost/templates/compose.yml.j2 @@ -15,6 +15,7 @@ command: ["/mattermost/bin/mattermost", "server"] {{ lookup('container_healthcheck', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} diff --git a/roles/web-app-mediawiki/meta/services.yml b/roles/web-app-mediawiki/meta/services.yml index 96e46688ed..3e956b9cb8 100644 --- a/roles/web-app-mediawiki/meta/services.yml +++ b/roles/web-app-mediawiki/meta/services.yml @@ -35,8 +35,8 @@ peertube: shared: "{{ 'web-app-peertube' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — DB engine, no Playwright surface (role-local integration tests) mariadb: bond: 1 @@ -68,6 +68,7 @@ mediawiki: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta cpus: "0.5" diff --git a/roles/web-app-mobilizon/meta/services.yml b/roles/web-app-mobilizon/meta/services.yml index 83cc31ab45..02e75b269c 100644 --- a/roles/web-app-mobilizon/meta/services.yml +++ b/roles/web-app-mobilizon/meta/services.yml @@ -19,8 +19,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" postgres: bond: 1 enabled: true # nocheck: dynamic-flag @@ -50,6 +50,7 @@ mobilizon: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta cpus: "0.2" diff --git a/roles/web-app-moodle/meta/services.yml b/roles/web-app-moodle/meta/services.yml index e127f00332..cc370e27dd 100644 --- a/roles/web-app-moodle/meta/services.yml +++ b/roles/web-app-moodle/meta/services.yml @@ -23,8 +23,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" ldap: bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" diff --git a/roles/web-app-n8n/meta/services.yml b/roles/web-app-n8n/meta/services.yml index b68876212d..a7ad1c3b40 100644 --- a/roles/web-app-n8n/meta/services.yml +++ b/roles/web-app-n8n/meta/services.yml @@ -15,8 +15,8 @@ sso: - "{{ lookup('rbac_group_path', application_id='web-app-n8n', role='mcp') }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" logout: bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" @@ -88,6 +88,7 @@ n8n: run_after: - web-app-keycloak - web-app-matomo + - web-app-stalwart - web-app-mailu lifecycle: beta # nocheck: dynamic-flag diff --git a/roles/web-app-n8n/templates/compose.yml.j2 b/roles/web-app-n8n/templates/compose.yml.j2 index c654cb17df..a8a49d0aa9 100644 --- a/roles/web-app-n8n/templates/compose.yml.j2 +++ b/roles/web-app-n8n/templates/compose.yml.j2 @@ -8,6 +8,7 @@ {{ lookup('container_ports', ['n8n', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('compose_volumes') }} diff --git a/roles/web-app-nextcloud/files/playwright/addons/xwiki.spec.js b/roles/web-app-nextcloud/files/playwright/addons/xwiki.spec.js index abef8f95c7..1412e16828 100644 --- a/roles/web-app-nextcloud/files/playwright/addons/xwiki.spec.js +++ b/roles/web-app-nextcloud/files/playwright/addons/xwiki.spec.js @@ -1,9 +1,13 @@ const { test, expect } = require("@playwright/test"); const { resolveTimeout } = require("../timeouts"); const { skipUnlessAddonEnabled } = require("../addon-gating"); +const { decodeDotenvQuotedValue } = require("../personas"); const shared = require("../_shared"); const { gotoOnion } = require("../personas"); +// Partner host comes from the env file — never hardcode the stack domain. +const xwikiPartnerHost = decodeDotenvQuotedValue(process.env.XWIKI_PARTNER_HOST || ""); + test.use({ ignoreHTTPSErrors: true }); test("xwiki addon: Nextcloud admin XWiki app renders and is coupled to the partner instance", async ({ browser }) => { @@ -67,6 +71,10 @@ test("xwiki addon: Nextcloud admin XWiki app renders and is coupled to the partn "the configured XWiki instance must be the partner instance, not Nextcloud itself" ).not.toBe(nextcloudHost); const expectedXwikiHost = new URL(shared.env.xwikiBaseUrl).host; + expect( + xwikiPartnerHost, + "XWIKI_PARTNER_HOST must be set in the Playwright env file" + ).toBeTruthy(); expect( instanceHost, "the XWiki instances appValue must point at the deployed XWiki partner host" diff --git a/roles/web-app-nextcloud/meta/services.yml b/roles/web-app-nextcloud/meta/services.yml index db2238cb5c..07dd3682d7 100644 --- a/roles/web-app-nextcloud/meta/services.yml +++ b/roles/web-app-nextcloud/meta/services.yml @@ -29,8 +29,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: shared # nocheck: dynamic-flag # nocheck: playwright-service-flag: DB engine, covered by role-local integration tests diff --git a/roles/web-app-nextcloud/templates/playwright.env.j2 b/roles/web-app-nextcloud/templates/playwright.env.j2 index 082f982584..a701c33136 100644 --- a/roles/web-app-nextcloud/templates/playwright.env.j2 +++ b/roles/web-app-nextcloud/templates/playwright.env.j2 @@ -81,5 +81,7 @@ SSO_SERVICE_ENABLED={{ NEXTCLOUD_SSO_ENABLED | string | lower }} XWIKI_BASE_URL={{ (lookup('tls', 'web-app-xwiki', 'url.base') if (lookup('config', application_id, 'services.xwiki.enabled') | bool) else '') | dotenv_quote }} +XWIKI_PARTNER_HOST={{ lookup('domain', 'web-app-xwiki') | dotenv_quote }} + {{ lookup('addon_env_flags', application_id) }} diff --git a/roles/web-app-odoo/meta/services.yml b/roles/web-app-odoo/meta/services.yml index 93db7b28ba..c094f81122 100644 --- a/roles/web-app-odoo/meta/services.yml +++ b/roles/web-app-odoo/meta/services.yml @@ -22,8 +22,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — DB engine, covered by role-local integration tests postgres: bond: 1 @@ -62,6 +62,7 @@ odoo: service_key: odoo run_after: - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta # nocheck: dynamic-flag diff --git a/roles/web-app-odoo/templates/compose.yml.j2 b/roles/web-app-odoo/templates/compose.yml.j2 index 3e65cebcbc..2bd182d5dc 100644 --- a/roles/web-app-odoo/templates/compose.yml.j2 +++ b/roles/web-app-odoo/templates/compose.yml.j2 @@ -20,6 +20,7 @@ ['odoo', 'http', DOCKER_BIND_HOST], ['odoo', 'websocket', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} diff --git a/roles/web-app-opencloud/meta/services.yml b/roles/web-app-opencloud/meta/services.yml index b41e09c5f9..876b7ebe19 100644 --- a/roles/web-app-opencloud/meta/services.yml +++ b/roles/web-app-opencloud/meta/services.yml @@ -24,8 +24,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" opencloud: healthcheck: flavor: wget diff --git a/roles/web-app-opencloud/templates/compose.yml.j2 b/roles/web-app-opencloud/templates/compose.yml.j2 index be87a88775..3bd99b5125 100644 --- a/roles/web-app-opencloud/templates/compose.yml.j2 +++ b/roles/web-app-opencloud/templates/compose.yml.j2 @@ -11,6 +11,7 @@ {% include 'roles/sys-svc-container/templates/base.yml.j2' %} {{ lookup('container_ports', ['opencloud', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('compose_volumes') }} diff --git a/roles/web-app-openproject/meta/services.yml b/roles/web-app-openproject/meta/services.yml index ed07b9cf66..b8906a2cfc 100644 --- a/roles/web-app-openproject/meta/services.yml +++ b/roles/web-app-openproject/meta/services.yml @@ -19,8 +19,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — DB engine, covered by role-local integration tests postgres: bond: 1 diff --git a/roles/web-app-openproject/templates/compose.yml.j2 b/roles/web-app-openproject/templates/compose.yml.j2 index 502751da2b..cd4001b730 100644 --- a/roles/web-app-openproject/templates/compose.yml.j2 +++ b/roles/web-app-openproject/templates/compose.yml.j2 @@ -34,6 +34,7 @@ x-op-app: &app {% include 'roles/sys-svc-container/templates/base.yml.j2' %} command: ["{{ OPENPROJECT_AWAIT_MIGRATIONS_TARGET }}", "{{ OPENPROJECT_MIGRATION_WAIT_SECONDS }}", "{{ OPENPROJECT_MIGRATION_POLL_SECONDS }}", "./docker/prod/web"] {{ lookup('compose_only', 'container_name', OPENPROJECT_WEB_CONTAINER) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_depends_on', application_id, extra={ 'cache': {'condition': 'service_started'}, @@ -48,6 +49,7 @@ x-op-app: &app {% include 'roles/sys-svc-container/templates/base.yml.j2' %} command: ["{{ OPENPROJECT_AWAIT_MIGRATIONS_TARGET }}", "{{ OPENPROJECT_MIGRATION_WAIT_SECONDS }}", "{{ OPENPROJECT_MIGRATION_POLL_SECONDS }}", "./docker/prod/worker"] {{ lookup('compose_only', 'container_name', OPENPROJECT_WORKER_CONTAINER) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_depends_on', application_id, extra={ 'cache': {'condition': 'service_started'}, @@ -61,6 +63,7 @@ x-op-app: &app {% include 'roles/sys-svc-container/templates/base.yml.j2' %} command: "./docker/prod/cron" {{ lookup('compose_only', 'container_name', OPENPROJECT_CRON_CONTAINER) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_depends_on', application_id, extra={ 'cache': {'condition': 'service_started'}, @@ -75,6 +78,7 @@ x-op-app: &app {% include 'roles/sys-svc-container/templates/base.yml.j2' %} command: "./docker/prod/seeder" {{ lookup('compose_only', 'container_name', OPENPROJECT_SEEDER_CONTAINER) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} diff --git a/roles/web-app-opentalk/meta/services.yml b/roles/web-app-opentalk/meta/services.yml index 2053628278..bc6067a322 100644 --- a/roles/web-app-opentalk/meta/services.yml +++ b/roles/web-app-opentalk/meta/services.yml @@ -26,8 +26,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — infrastructural sub-component (TURN/STUN) coturn: bond: 1 diff --git a/roles/web-app-opentalk/meta/volumes.yml b/roles/web-app-opentalk/meta/volumes.yml index 1fbcdb6011..866200de2e 100644 --- a/roles/web-app-opentalk/meta/volumes.yml +++ b/roles/web-app-opentalk/meta/volumes.yml @@ -41,6 +41,14 @@ livekit_yaml: - service: livekit target: /etc/livekit/livekit.yaml +rabbitmq_conf: + type: config + source: "{{ [lookup('container', 'web-app-opentalk', 'directories.volumes'), 'rabbitmq.conf'] | path_join }}" + mode: "0444" + mounts: + - service: rabbit + target: /etc/rabbitmq/conf.d/10-deprecated-features.conf + recorder_toml: type: config source: "{{ [lookup('container', 'web-app-opentalk', 'directories.volumes'), 'recorder.toml'] | path_join }}" diff --git a/roles/web-app-opentalk/tasks/01_manager_ops.yml b/roles/web-app-opentalk/tasks/01_manager_ops.yml index ebdbecb1bd..8b1be21517 100644 --- a/roles/web-app-opentalk/tasks/01_manager_ops.yml +++ b/roles/web-app-opentalk/tasks/01_manager_ops.yml @@ -15,6 +15,14 @@ notify: - compose-up +- name: "📄 Render RabbitMQ config '{{ application_id }}'" + stack_host_template: + src: rabbitmq.conf.j2 + dest: "{{ [lookup('container', application_id, 'directories.volumes'), 'rabbitmq.conf'] | path_join }}" + mode: "0644" + notify: + - compose-up + - name: "📄 Render OpenTalk recorder config '{{ application_id }}'" stack_host_template: src: recorder.toml.j2 diff --git a/roles/web-app-opentalk/templates/rabbitmq.conf.j2 b/roles/web-app-opentalk/templates/rabbitmq.conf.j2 new file mode 100644 index 0000000000..ca9f50c72a --- /dev/null +++ b/roles/web-app-opentalk/templates/rabbitmq.conf.j2 @@ -0,0 +1,6 @@ +# The pinned recorder ({{ lookup('config', application_id, 'services.recorder.version') }}) declares its task queue as +# transient and non-exclusive. RabbitMQ 4 refuses that by default and answers +# the declare with an AMQP hard error, so the recorder never consumes and +# retries forever. Permitting it keeps recordings working until the recorder +# leaves the rabbitmq transport; see the version pin in meta/services.yml. +deprecated_features.permit.transient_nonexcl_queues = true diff --git a/roles/web-app-openwebui/meta/services.yml b/roles/web-app-openwebui/meta/services.yml index 2f63bd2d82..865f73cc06 100644 --- a/roles/web-app-openwebui/meta/services.yml +++ b/roles/web-app-openwebui/meta/services.yml @@ -118,8 +118,8 @@ jupyter: email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" prometheus: bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-openwebui/templates/compose.yml.j2 b/roles/web-app-openwebui/templates/compose.yml.j2 index 9b85416e55..61bdab0a96 100644 --- a/roles/web-app-openwebui/templates/compose.yml.j2 +++ b/roles/web-app-openwebui/templates/compose.yml.j2 @@ -6,6 +6,7 @@ {{ lookup('compose_only', 'container_name', OPENWEBUI_CONTAINER) }} {{ lookup('container_ports', ['openwebui', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('compose_networks') }} diff --git a/roles/web-app-peertube/meta/services.yml b/roles/web-app-peertube/meta/services.yml index 9012a3438f..e9aab4e5c1 100644 --- a/roles/web-app-peertube/meta/services.yml +++ b/roles/web-app-peertube/meta/services.yml @@ -18,8 +18,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: shared # nocheck: dynamic-flag # nocheck: playwright-service-flag — DB engine, covered by role-local integration tests diff --git a/roles/web-app-penpot/meta/services.yml b/roles/web-app-penpot/meta/services.yml index 0e590af69b..6b6f3de954 100644 --- a/roles/web-app-penpot/meta/services.yml +++ b/roles/web-app-penpot/meta/services.yml @@ -13,8 +13,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" sso: bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" @@ -74,6 +74,7 @@ penpot: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta frontend: diff --git a/roles/web-app-pgadmin/meta/services.yml b/roles/web-app-pgadmin/meta/services.yml index 20b16398f4..4184791bf4 100644 --- a/roles/web-app-pgadmin/meta/services.yml +++ b/roles/web-app-pgadmin/meta/services.yml @@ -26,6 +26,7 @@ pgadmin: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta cpus: "0.2" @@ -46,8 +47,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" postgres: bond: 1 enabled: true # nocheck: dynamic-flag diff --git a/roles/web-app-pgadmin/templates/compose.yml.j2 b/roles/web-app-pgadmin/templates/compose.yml.j2 index fdc14ec7e0..938079287c 100644 --- a/roles/web-app-pgadmin/templates/compose.yml.j2 +++ b/roles/web-app-pgadmin/templates/compose.yml.j2 @@ -7,6 +7,7 @@ {% include 'roles/sys-svc-container/templates/base.yml.j2' %} {{ lookup('container_ports', ['pgadmin', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} diff --git a/roles/web-app-pixelfed/files/playwright/playwright.spec.js b/roles/web-app-pixelfed/files/playwright/playwright.spec.js index f7da6c81c7..5b13289c5f 100644 --- a/roles/web-app-pixelfed/files/playwright/playwright.spec.js +++ b/roles/web-app-pixelfed/files/playwright/playwright.spec.js @@ -13,10 +13,10 @@ test.use({ test.beforeAll(shared.beforeAll); -test.afterAll(shared.afterAll); - test.beforeEach(shared.beforeEach); +test.afterAll(shared.afterAll); + require("./test-oidc-login-biber").register(shared); require("./test-oidc-login-administrator").register(shared); require("./test-native-login-administrator").register(shared); diff --git a/roles/web-app-pixelfed/meta/services.yml b/roles/web-app-pixelfed/meta/services.yml index 60f68c1f96..ae553cae61 100644 --- a/roles/web-app-pixelfed/meta/services.yml +++ b/roles/web-app-pixelfed/meta/services.yml @@ -18,8 +18,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: shared # nocheck: dynamic-flag # nocheck: playwright-service-flag — DB engine, covered by role-local integration tests diff --git a/roles/web-app-pretix/meta/services.yml b/roles/web-app-pretix/meta/services.yml index bcccb16e2c..fa74ddee1b 100644 --- a/roles/web-app-pretix/meta/services.yml +++ b/roles/web-app-pretix/meta/services.yml @@ -19,8 +19,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: shared # nocheck: dynamic-flag # nocheck: playwright-service-flag — Redis cache, no Playwright surface @@ -71,6 +71,7 @@ pretix: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta # nocheck: playwright-service-flag — CSS consumer, no Playwright surface (provider: web-svc-css) diff --git a/roles/web-app-pretix/templates/compose.yml.j2 b/roles/web-app-pretix/templates/compose.yml.j2 index d8c3f37d31..faae214679 100644 --- a/roles/web-app-pretix/templates/compose.yml.j2 +++ b/roles/web-app-pretix/templates/compose.yml.j2 @@ -15,6 +15,7 @@ {{ lookup('container_healthcheck', service_name) | indent(4) }} {% include 'roles/sys-svc-container/templates/base.yml.j2' %} {{ lookup('container_depends_on', application_id) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% if PRETIX_MCP_ENABLED | bool %} diff --git a/roles/web-app-prometheus/meta/services.yml b/roles/web-app-prometheus/meta/services.yml index 6d5ac81299..209aed5785 100644 --- a/roles/web-app-prometheus/meta/services.yml +++ b/roles/web-app-prometheus/meta/services.yml @@ -24,8 +24,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: playwright-service-flag — css consumer, no Playwright surface (provider: web-svc-css) css: bond: 1 diff --git a/roles/web-app-prometheus/meta/volumes.yml b/roles/web-app-prometheus/meta/volumes.yml index b64eb9283d..5d6968ecf3 100644 --- a/roles/web-app-prometheus/meta/volumes.yml +++ b/roles/web-app-prometheus/meta/volumes.yml @@ -97,3 +97,11 @@ host_proc: mounts: - service: node-exporter target: /host/proc + +metrics_textfile: + type: bind + source: "{{ DIR_METRICS_TEXTFILE }}" + read_only: true + mounts: + - service: node-exporter + target: /host/textfile diff --git a/roles/web-app-prometheus/tasks/00_core.yml b/roles/web-app-prometheus/tasks/00_core.yml index b2b917a558..7a9d60c6fc 100644 --- a/roles/web-app-prometheus/tasks/00_core.yml +++ b/roles/web-app-prometheus/tasks/00_core.yml @@ -12,6 +12,12 @@ vars: compose_handlers_flush: false +- name: "📁 Ensure the node-exporter textfile directory exists" + ansible.builtin.file: + path: "{{ DIR_METRICS_TEXTFILE }}" + state: directory + mode: "0755" + - name: "📄 Render Prometheus config files (local + swarm peers)" ansible.builtin.include_tasks: file: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/render_replicated_templates.yml') }}" diff --git a/roles/web-app-prometheus/templates/compose.yml.j2 b/roles/web-app-prometheus/templates/compose.yml.j2 index 57ea6e1c2b..e8ae4a8214 100644 --- a/roles/web-app-prometheus/templates/compose.yml.j2 +++ b/roles/web-app-prometheus/templates/compose.yml.j2 @@ -69,6 +69,7 @@ - "--path.rootfs={{ lookup('volume', 'web-app-prometheus', 'host_rootfs').target }}" - "--path.sysfs={{ lookup('volume', 'web-app-prometheus', 'host_sys').targets['node-exporter'] }}" - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)" + - "--collector.textfile.directory={{ lookup('volume', 'web-app-prometheus', 'metrics_textfile').targets['node-exporter'] }}" {% if PROMETHEUS_MCP_ENABLED | bool %} {% set service_name = PROMETHEUS_MCP_SERVICE_KEY %} diff --git a/roles/web-app-prometheus/templates/configuration/alert_rules.yml.j2 b/roles/web-app-prometheus/templates/configuration/alert_rules.yml.j2 index 0c6f9a139d..f31db0db30 100644 --- a/roles/web-app-prometheus/templates/configuration/alert_rules.yml.j2 +++ b/roles/web-app-prometheus/templates/configuration/alert_rules.yml.j2 @@ -15,6 +15,30 @@ groups: {{ '{{' }} $labels.instance {{ '}}' }} has been unreachable for more than 2 minutes. Check the application container and upstream connectivity. + - alert: TlsCertificateExpiringSoon + expr: (probe_ssl_earliest_cert_expiry{job="blackbox-healthz"} - time()) < 14 * 86400 + for: 30m + labels: + severity: warning + annotations: + summary: "TLS certificate for {{ '{{' }} $labels.app {{ '}}' }} expires in less than 14 days" + description: > + The certificate served at {{ '{{' }} $labels.instance {{ '}}' }} + (app: {{ '{{' }} $labels.app {{ '}}' }}) expires soon. + Check certificate renewal and sys-ctl-mtn-cert-deploy. + + - alert: TlsCertificateExpiryCritical + expr: (probe_ssl_earliest_cert_expiry{job="blackbox-healthz"} - time()) < 3 * 86400 + for: 30m + labels: + severity: critical + annotations: + summary: "TLS certificate for {{ '{{' }} $labels.app {{ '}}' }} expires in less than 3 days" + description: > + The certificate served at {{ '{{' }} $labels.instance {{ '}}' }} + (app: {{ '{{' }} $labels.app {{ '}}' }}) is about to expire. + Renew immediately; clients will start failing at expiry. + - alert: AppHighLatency expr: > histogram_quantile(0.95, diff --git a/roles/web-app-prometheus/templates/configuration/alertmanager.yml.j2 b/roles/web-app-prometheus/templates/configuration/alertmanager.yml.j2 index d2dd562c75..e7e6922ddd 100644 --- a/roles/web-app-prometheus/templates/configuration/alertmanager.yml.j2 +++ b/roles/web-app-prometheus/templates/configuration/alertmanager.yml.j2 @@ -5,6 +5,7 @@ global: smtp_smarthost: "{{ email.host }}:{{ email.port }}" smtp_from: "{{ email['from'] }}" + smtp_hello: "{{ email.domain }}" smtp_require_tls: {{ email.tls | bool | lower }} {% if email.auth | bool %} smtp_auth_username: "{{ email.username }}" diff --git a/roles/web-app-semaphore/meta/services.yml b/roles/web-app-semaphore/meta/services.yml index 3333110359..f12e6790d8 100644 --- a/roles/web-app-semaphore/meta/services.yml +++ b/roles/web-app-semaphore/meta/services.yml @@ -23,8 +23,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" css: bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" diff --git a/roles/web-app-shopware/meta/services.yml b/roles/web-app-shopware/meta/services.yml index 6211b8492d..485d6c2c09 100644 --- a/roles/web-app-shopware/meta/services.yml +++ b/roles/web-app-shopware/meta/services.yml @@ -25,8 +25,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" mariadb: bond: 1 enabled: true # nocheck: dynamic-flag @@ -63,6 +63,7 @@ shopware: http: 8060 run_after: - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta cpus: "1.0" diff --git a/roles/web-app-snipe-it/meta/services.yml b/roles/web-app-snipe-it/meta/services.yml index 2f144e8bf2..15dab11482 100644 --- a/roles/web-app-snipe-it/meta/services.yml +++ b/roles/web-app-snipe-it/meta/services.yml @@ -19,8 +19,8 @@ matomo: shared: "{{ 'web-app-matomo' in group_names }}" email: bond: 1 - enabled: "{{ 'web-app-mailu' in group_names }}" - shared: "{{ 'web-app-mailu' in group_names }}" + enabled: "{{ MAIL_PROVIDER in group_names }}" + shared: "{{ MAIL_PROVIDER in group_names }}" # nocheck: shared # nocheck: dynamic-flag # nocheck: playwright-service-flag — cache engine, no Playwright surface @@ -78,6 +78,7 @@ snipe-it: run_after: - web-app-matomo - web-app-keycloak + - web-app-stalwart - web-app-mailu lifecycle: beta sso: diff --git a/roles/web-app-snipe-it/templates/compose.yml.j2 b/roles/web-app-snipe-it/templates/compose.yml.j2 index 08b7272ced..ae2586f59b 100644 --- a/roles/web-app-snipe-it/templates/compose.yml.j2 +++ b/roles/web-app-snipe-it/templates/compose.yml.j2 @@ -12,6 +12,7 @@ {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('container_ports', ['snipe-it', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} + {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_networks') | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} diff --git a/roles/web-app-socialhome/meta/services.yml b/roles/web-app-socialhome/meta/services.yml index dd3ee19336..e6aaaf86d0 100644 --- a/roles/web-app-socialhome/meta/services.yml +++ b/roles/web-app-socialhome/meta/services.yml @@ -13,7 +13,7 @@ socialhome: # nocheck: default-placement-manager embedded sqlite in a node-local volume; no clustered storage placement: manager image: ghcr.io/social-home-io/socialhome - version: "2026.9.29" + version: "2026.10.2" ports: local: http: 8089 diff --git a/roles/web-app-stalwart/README.md b/roles/web-app-stalwart/README.md new file mode 100644 index 0000000000..41b928bece --- /dev/null +++ b/roles/web-app-stalwart/README.md @@ -0,0 +1,178 @@ +# Stalwart Mail Server + +## Description + +Runs [Stalwart](https://stalw.art/) — a secure, all-in-one mail and +collaboration server — as the platform's email provider. A single hardened +service speaks **SMTP, Submission, IMAP, POP3, JMAP, ManageSieve, CalDAV, +CardDAV and WebDAV**, with a **built-in spam filter**, **DKIM/DMARC** signing +and a **WebAdmin + REST management API**. + +This role replaces the deprecated [`web-app-mailu`](../web-app-mailu/) role. +Applications send mail through the same provider-agnostic abstraction +(`lookup('email')` / [`sys-svc-mail`](../sys-svc-mail/)) — no consumer changes +are needed beyond the provider repoint. + +## Overview + +Compared with Mailu's multi-container stack, Stalwart collapses the mail server +into one binary. This role runs: + +| Container | Purpose | +|-----------|---------| +| `stalwart` | SMTP/IMAP/JMAP/POP3/Sieve/DAV + built-in spam filter + WebAdmin/REST API | +| `webmail` (Roundcube) | Browser webmail (parity with Mailu) | +| `clamav` | Attachment antivirus, called as an SMTP DATA-stage milter | +| `postgres` *(shared)* | Account / mail / metadata store | + +Dynamic state (domains, accounts, passwords, DKIM, certificates) is administered +at runtime via the JMAP management API; `config.json` only bootstraps the data +store. Spam filtering is built into Stalwart; **antivirus** is provided by the +`clamav` container (registered via `x:MtaMilter`, `services.clamav.enabled` — +set it to `false` to run spam-only). Infected mail is rejected at DATA. + +```mermaid +flowchart LR + subgraph edge["Front proxy (TLS)"] + MAIL_VHOST["mail. (WebAdmin)"] + WEBMAIL_VHOST["webmail. (Roundcube)"] + end + + subgraph stack["web-app-stalwart compose stack"] + STALWART["stalwart
SMTP/IMAP/JMAP/POP3/Sieve/DAV
+ WebAdmin + REST API"] + WEBMAIL["webmail (Roundcube)"] + CLAMAV["clamav (DATA-stage milter)"] + PG[("postgres
(shared platform DB)")] + end + + KC["Keycloak
OIDC directory + SSO"] + APPS["Platform apps
(no-reply bot via msmtp)"] + WORLD(("Internet
ports 25/465/587/143/993/...")) + + MAIL_VHOST --> STALWART + WEBMAIL_VHOST --> WEBMAIL + WEBMAIL -- "IMAP 993 / SMTP 465
XOAUTH2 (stalwart-webmail client)" --> STALWART + STALWART -- "milter" --> CLAMAV + STALWART --- PG + STALWART -- "OIDC directory
(token validation)" --> KC + MAIL_VHOST -. "SPA login: PKCE
(stalwart-webui client)" .-> KC + WEBMAIL_VHOST -. "OAuth login" .-> KC + APPS -- "STARTTLS :25, no AUTH
(SSO relay, trusted networks)" --> STALWART + WORLD <-- "public mail ports
(only when active MAIL_PROVIDER)" --> STALWART +``` + +## SSO (Keycloak / OpenID Connect) + +When `web-app-keycloak` is present the role joins the shared Keycloak client and +delegates **interactive** authentication to Keycloak: + +- **WebAdmin + IMAP/SMTP/JMAP** authenticate against an **OIDC directory** + (`tasks/06_oidc.yml`): Stalwart validates Keycloak-issued tokens, matching the + `preferred_username` claim to the provisioned account. +- **Roundcube** logs users in over OAuth2 and talks to Stalwart with **XOAUTH2** + (`templates/roundcube-oauth.inc.php.j2`). + +**Design constraint (validated against the live JMAP schema):** Stalwart's +authentication directory is *Internal XOR one external directory* — there is no +chaining or fallback. Enabling SSO therefore **disables password submission** +for every account, including the machine `no-reply` account. To keep outbound +notifications working, the role: + +1. Widens `x:MtaStageRcpt.allowRelaying` to trust the internal Docker networks + (`STALWART_TRUSTED_NETWORKS`), so the bot relays without SMTP AUTH; and +2. Self-declares `services.sso.oidc.submission_via_relay: true`, which makes + [`plugins/lookup/email.py`](../../plugins/lookup/email.py) switch the + `no-reply` client to unauthenticated STARTTLS relay on port 25. + +With SSO disabled the role uses the Internal directory and password submission, +exactly as before. Mailu keeps password submission in both modes and does not +set `submission_via_relay`. + +## Calendar & Contacts (CalDAV / CardDAV / WebDAV) + +Stalwart serves DAV natively on the mail HTTP listener — no extra container +(Mailu used a separate Radicale service). Clients auto-discover via +`https://mail./.well-known/{caldav,carddav}`, which redirect to +`https://mail./dav/cal` and `/dav/card`. Authenticate with the mailbox +account (or the Keycloak SSO token when SSO is enabled). + +## Design Decisions + +- **Client ports are implicit-TLS only (465/993/995).** Stalwart's default + configuration binds no STARTTLS listeners, so the role neither publishes + 587/143/110 nor advertises them via SRV records. +- **The WebUI (WebAdmin) bundle is staged by the controller.** Stalwart fetches + it from GitHub on first boot, which fails behind restricted egress; the + controller downloads it into a host-level cache (matrix rounds purge the + instance dir, and per-round downloads hit GitHub rate limits) and bind-mounts + it read-only. `webui_version` in `meta/services.yml` MUST be bumped together + with the Stalwart `version`. +- **The embedded PostgreSQL pins the plain `postgres` image** — the platform's + postgis default defeats baudolo's substring database detection and would + degrade backups to torn live-file snapshots. `backup.project_hard_restart` + brings the whole project back through one coherent restart after baudolo's + per-volume stop/start cycles. +- **ClamAV runs fail-open with short milter timeouts** so mail keeps flowing + while clamd warms its signature database instead of blocking the DATA stage. +- **`config.json` is bootstrap-only** (data store selection); all other + configuration lives in the database and is provisioned over JMAP + (`templates/jmap/*.json.j2`). +- **Under SSO, trusted internal networks relay without SMTP AUTH** — the + no-reply bot has no password once the auth directory is OIDC. + +## Features + +- All-in-one mail server (SMTP/IMAP/JMAP/POP3/ManageSieve) +- CalDAV / CardDAV / WebDAV collaboration +- Built-in spam filtering +- ClamAV antivirus as an SMTP DATA-stage milter (`services.clamav.enabled: false` for spam-only) +- DKIM signing with automatic key management; SPF / DMARC published in DNS +- OpenID Connect SSO via Keycloak +- Roundcube webmail +- PostgreSQL backing store + +## Migration from Mailu + +The provider cutover is an inventory change; mailbox data moves with it when the migration switch is on. + +> **Compose only.** The import reads the legacy maildir volume on the stack host and submits over the host-published IMAPS port, neither of which holds under swarm. Migrate in compose mode, then move the stack to swarm. + +1. Keep `web-app-mailu` in the host's groups and add `web-app-stalwart` — both MUST be present so Mailu re-renders as a legacy instance (`legacy-mail.`, no public mail ports) and releases `mail.` to Stalwart. +2. Remove `MAIL_PROVIDER` from the inventory (Stalwart is the default). +3. Set `applications["web-app-stalwart"].services.stalwart.migration.import_mailu: true` and run the full deploy. The role then reads Mailu's Dovecot Maildir volume and imports every inventory account's messages via IMAP APPEND, preserving folders, flags and internal dates. Re-runs are idempotent (Message-ID dedup), so the switch may stay on across deploys; turn it off once Mailu is retired. + The import authenticates per account with the inventory password, so it MUST run while `services.sso.enabled` is false. An OIDC directory replaces password login outright (see [`tasks/06_oidc.yml`](tasks/06_oidc.yml)), and every IMAP login then fails with `AUTHENTICATIONFAILED`. Migrate first, enable SSO afterwards. +4. Accounts and aliases are provisioned from the inventory by this role; mailboxes created only inside Mailu's admin UI MUST be added to the inventory first. Sieve filters and CalDAV/CardDAV data are out of scope. +5. Non-Cloudflare DNS: publish the new DKIM TXT record reported by the deploy; MX and A records keep their hostname. + +The cutover is covered end to end by [`files/test/test.sh`](files/test/test.sh), which runs as this role's CLI test whenever `web-app-mailu` is co-deployed — matrix **variant 1**, the round where SSO is off, because the import authenticates with per-account passwords. The whole scenario runs inside the deployed host and needs nothing from the environment around it: it stores mail in both directions in the legacy Mailu, confirms it landed in the maildir volume, runs [`files/python/migrate_from_mailu.py`](files/python/migrate_from_mailu.py) — the same script and argv a real cutover uses — and then asserts over IMAP that the stored mail survived and that live mail still flows. No redeploy is involved; the cutover is a docker-level data move. + +## Verifying a production server + +Deploying against a publicly reachable mail server additionally asserts that server's live **hard facts** — the records and listeners either match the deployed configuration or they do not. There is no mail-flow scenario here; that is what the suites above are for. + +Set `INFINITO_MAIL_PRODUCTION_HOST` to the server's public FQDN (in CI, a GitHub repository variable of the same name; empty disables the whole check). Optionally set `INFINITO_MAIL_PRODUCTION_RESOLVER` to the address of a public resolver so the records are read from the outside view instead of `/etc/resolv.conf`, which matters on a host whose own resolver answers from a split-horizon zone. + +[`files/python/production_facts.py`](files/python/production_facts.py) then checks, read-only: + +| Fact | Assertion | +|---|---| +| A / AAAA | the mail host resolves | +| MX | the zone's only MX is the mail host (the role publishes it `solo`) | +| SPF | exactly one `v=spf1` record, authorising `mx` or `a:` | +| DMARC | `_dmarc` publishes a `v=DMARC1` record with a policy | +| DKIM | the selector this deploy read back publishes a non-empty `v=DKIM1` key | +| SRV | `_submissions`/465, `_imaps`/993, `_pop3s`/995 point at the mail host | +| PTR | rDNS does not contradict the mail host (absent → warning; rDNS is provisioned for Hetzner only) | +| SMTP | port 25 greets with `220` carrying the FQDN, proving the `rejectNonFqdn` identity | + +The same variable points [`sys-ctl-mtn-cert-deploy`](../sys-ctl-mtn-cert-deploy/)'s remote check at `:465` and `:993`, so the live TLS chain and its remaining validity are verified alongside. A misconfigured production server therefore fails the deploy instead of surfacing as an outage. + +## Further Reading + +- [Stalwart documentation](https://stalw.art/docs) +- [`sys-svc-mail`](../sys-svc-mail/) — how applications send mail +- [`plugins/lookup/email.py`](../../plugins/lookup/email.py) — the email abstraction + +> **Note:** Stalwart's JMAP object schema is version-sensitive. Pin +> `services.stalwart.version`; the provisioning payloads in `tasks/` were +> validated against that release's `GET /api/schema`. diff --git a/roles/web-app-stalwart/files/playwright/.nocheck b/roles/web-app-stalwart/files/playwright/.nocheck new file mode 100644 index 0000000000..9ceebc5e82 --- /dev/null +++ b/roles/web-app-stalwart/files/playwright/.nocheck @@ -0,0 +1 @@ +# nocheck: role-files-layout - Playwright discovers this suite by path; the runner copies the tree as it stands and the specs import their helpers relatively. Sorting by extension would break discovery. diff --git a/roles/web-app-stalwart/files/playwright/env.js b/roles/web-app-stalwart/files/playwright/env.js new file mode 100644 index 0000000000..1d0acb39b1 --- /dev/null +++ b/roles/web-app-stalwart/files/playwright/env.js @@ -0,0 +1,34 @@ +const { decodeDotenvQuotedValue } = require("./personas"); + +const appBaseUrl = decodeDotenvQuotedValue(process.env.APP_BASE_URL || "").replace(/\/+$/, ""); +const webmailBaseUrl = decodeDotenvQuotedValue(process.env.WEBMAIL_BASE_URL || "").replace(/\/+$/, ""); +const canonicalDomain = decodeDotenvQuotedValue(process.env.CANONICAL_DOMAIN || ""); +const oidcIssuerUrl = decodeDotenvQuotedValue(process.env.OIDC_ISSUER_URL || ""); +const adminEmail = decodeDotenvQuotedValue(process.env.ADMIN_EMAIL || ""); +const adminUsername = decodeDotenvQuotedValue(process.env.ADMIN_USERNAME || ""); +const adminPassword = decodeDotenvQuotedValue(process.env.ADMIN_PASSWORD || ""); +const stalwartAdminUsername = decodeDotenvQuotedValue(process.env.STALWART_ADMIN_USERNAME || ""); +const stalwartAdminPassword = decodeDotenvQuotedValue(process.env.STALWART_ADMIN_PASSWORD || ""); +const biberEmail = decodeDotenvQuotedValue(process.env.BIBER_EMAIL || ""); +const biberUsername = decodeDotenvQuotedValue(process.env.BIBER_USERNAME || ""); +const biberPassword = decodeDotenvQuotedValue(process.env.BIBER_PASSWORD || ""); +const mapachePassword = decodeDotenvQuotedValue(process.env.MAPACHE_PASSWORD || ""); + +const expectedOidcAuthUrl = `${oidcIssuerUrl.replace(/\/$/, "")}/protocol/openid-connect/auth`; + +module.exports = { + appBaseUrl, + webmailBaseUrl, + canonicalDomain, + oidcIssuerUrl, + expectedOidcAuthUrl, + adminEmail, + adminUsername, + adminPassword, + stalwartAdminUsername, + stalwartAdminPassword, + biberEmail, + biberUsername, + biberPassword, + mapachePassword, +}; diff --git a/roles/web-app-stalwart/files/playwright/playwright.spec.js b/roles/web-app-stalwart/files/playwright/playwright.spec.js new file mode 100644 index 0000000000..5cc79363a7 --- /dev/null +++ b/roles/web-app-stalwart/files/playwright/playwright.spec.js @@ -0,0 +1,20 @@ +const { test, expect } = require("@playwright/test"); + +const { appBaseUrl, canonicalDomain } = require("./env"); + +// The runner collects only *.spec.js, so a test-*.js module missing from the +// list below is never run. +test.use({ ignoreHTTPSErrors: true }); + +test.beforeEach(() => { + expect(appBaseUrl, "APP_BASE_URL must be set").toBeTruthy(); + expect(canonicalDomain, "CANONICAL_DOMAIN must be set").toBeTruthy(); +}); + +require("./test-tls"); +require("./test-sso"); +require("./test-oidc-provisioning"); +require("./test-login-native"); +require("./test-mailflow"); +require("./test-onion-mailflow"); +require("./test-personas"); diff --git a/roles/web-app-stalwart/files/playwright/test-login-native.js b/roles/web-app-stalwart/files/playwright/test-login-native.js new file mode 100644 index 0000000000..8ecf9da8d0 --- /dev/null +++ b/roles/web-app-stalwart/files/playwright/test-login-native.js @@ -0,0 +1,69 @@ +const { test, expect } = require("@playwright/test"); + +const { safeIsEnabled, gotoOnion } = require("./personas"); +const { + appBaseUrl, + canonicalDomain, + oidcIssuerUrl, + stalwartAdminUsername, + stalwartAdminPassword, +} = require("./env"); +const { resolveTimeout } = require("./timeouts"); + +// WebAdmin native login: the variant where Keycloak is absent and Stalwart's own +// directory owns the credentials. Without this the sso=false variant deploys a +// login surface no spec ever exercises, because every other spec self-skips on sso. +// The account is Stalwart's own WebAdmin administrator (STALWART_ADMIN_USER), not the +// platform's administrator — the latter lives in the OIDC directory and is rejected here. +test("administrator: stalwart native login and logout (no sso)", async ({ page }) => { + test.skip( + safeIsEnabled("sso"), + "Native login is only exercised in the variant without Keycloak — when SSO is on the OIDC path owns the journey.", + ); + expect(stalwartAdminUsername, "STALWART_ADMIN_USERNAME must be set").toBeTruthy(); + expect(stalwartAdminPassword, "STALWART_ADMIN_PASSWORD must be set").toBeTruthy(); + + await gotoOnion(page, `${appBaseUrl}/`); + await expect.poll(() => page.url(), { timeout: resolveTimeout(30_000) }).toContain(`${canonicalDomain}/account/login`); + + const loginField = page + .getByRole("textbox", { name: /email|user|login/i }) + .or(page.locator("input[type='email'], input[type='text'], input[name='username'], input[name='email']")) + .first(); + await loginField.waitFor({ state: "visible", timeout: resolveTimeout(30_000) }); + await loginField.fill(stalwartAdminUsername); + + await page + .getByRole("button", { name: /continue/i }) + .or(page.locator("button[type='submit']")) + .first() + .click(); + + const passwordField = page.locator("input[type='password'], input[name='password']").first(); + await passwordField.waitFor({ state: "visible", timeout: resolveTimeout(30_000) }); + await passwordField.fill(stalwartAdminPassword); + await page + .getByRole("button", { name: /sign ?in|log ?in/i }) + .or(page.locator("button[type='submit']")) + .first() + .click(); + + await expect( + page.getByText(/invalid username or password/i) + ).toBeHidden({ timeout: resolveTimeout(15_000) }); + + await expect( + page.locator("nav, .sidebar, [class*='menu'], h1, h2").filter({ hasText: /dashboard|domains|account|settings|directory/i }).first() + ).toBeVisible({ timeout: resolveTimeout(30_000) }); + + if (oidcIssuerUrl) { + expect(page.url()).not.toContain(oidcIssuerUrl); + } + await expect.poll(() => page.url(), { timeout: resolveTimeout(15_000) }).toContain(canonicalDomain); + + const logout = page.locator("a[href*='logout'], button:has-text('Logout')").or(page.getByRole("link", { name: /logout/i })); + if (await logout.first().isVisible().catch(() => false)) { + await logout.first().click(); + await expect.poll(() => page.url(), { timeout: resolveTimeout(30_000) }).toContain("/account/login"); + } +}); diff --git a/roles/web-app-stalwart/files/playwright/test-mailflow.js b/roles/web-app-stalwart/files/playwright/test-mailflow.js new file mode 100644 index 0000000000..9980dd4bf9 --- /dev/null +++ b/roles/web-app-stalwart/files/playwright/test-mailflow.js @@ -0,0 +1,79 @@ +const { test, expect } = require("@playwright/test"); + +const { safeSkipUnlessEnabled, gotoOnion, recordVideoOptions } = require("./personas"); +const { roundcubeSsoLogin, roundcubeLogout, sendMail, waitForEmailInMailbox } = require("./webmail"); +const { + webmailBaseUrl, + adminEmail, + adminUsername, + adminPassword, + biberEmail, + biberUsername, + biberPassword, +} = require("./env"); +const { resolveTimeout, isSplitRealmOidc } = require("./timeouts"); + +/** Open a delivered message and confirm its body renders. */ +async function openMail(page, subject) { + const row = await waitForEmailInMailbox(page, webmailBaseUrl, subject, resolveTimeout(90_000)); + await expect(row).toBeVisible(); + await row.click(); + await expect( + page.locator("#messagecontframe, #mailview-right, .message-part").first() + ).toBeVisible({ timeout: resolveTimeout(15_000) }); +} + +async function leaveOpenDialogs(page) { + await gotoOnion(page, `${webmailBaseUrl}/?_task=mail&_mbox=INBOX`); + await page + .locator(".ui-widget-overlay") + .first() + .waitFor({ state: "detached", timeout: resolveTimeout(15_000) }) + .catch(() => {}); +} + +// biber <-> administrator round trip through the Roundcube webmail UI: biber writes to +// the administrator, the administrator reads it and writes back, and biber reads the +// answer. Login is via Keycloak SSO (Roundcube XOAUTH2 -> Stalwart), mirroring +// web-app-mailu. They are separate people: isolated browser contexts, and biber stays +// signed in so the answer lands in a session that never saw the outbound message. +test("stalwart: biber and the administrator exchange mail both ways", async ({ browser }, testInfo) => { + test.skip(isSplitRealmOidc(), "clearnet app with an onion OIDC issuer: unreachable from one browser"); + safeSkipUnlessEnabled("sso"); + expect(webmailBaseUrl, "WEBMAIL_BASE_URL must be set").toBeTruthy(); + expect(adminEmail, "ADMIN_EMAIL must be set").toBeTruthy(); + expect(biberEmail, "BIBER_EMAIL must be set").toBeTruthy(); + expect(biberPassword, "BIBER_PASSWORD must be set").toBeTruthy(); + expect(adminPassword, "ADMIN_PASSWORD must be set").toBeTruthy(); + + const outbound = `Playwright stalwart ${Date.now()}`; + const answer = `Playwright stalwart answer ${Date.now()}`; + const proxyServer = process.env.PLAYWRIGHT_PROXY; + const contextOptions = { + ignoreHTTPSErrors: true, + ...(proxyServer ? { proxy: { server: proxyServer } } : {}), + ...recordVideoOptions(testInfo), + }; + const biberContext = await browser.newContext(contextOptions); + const adminContext = await browser.newContext(contextOptions); + + try { + const biberPage = await biberContext.newPage(); + await roundcubeSsoLogin(biberPage, biberUsername, biberPassword); + await sendMail(biberPage, adminEmail, outbound, "Hello Administrator, this is an automated Playwright test email."); + + const adminPage = await adminContext.newPage(); + await roundcubeSsoLogin(adminPage, adminUsername, adminPassword); + await openMail(adminPage, outbound); + await leaveOpenDialogs(adminPage); + await sendMail(adminPage, biberEmail, answer, "Answer from the administrator, sent by the same Playwright run."); + await roundcubeLogout(adminPage); + + await openMail(biberPage, answer); + await leaveOpenDialogs(biberPage); + await roundcubeLogout(biberPage); + } finally { + await biberContext.close().catch(() => {}); + await adminContext.close().catch(() => {}); + } +}); diff --git a/roles/web-app-stalwart/files/playwright/test-oidc-provisioning.js b/roles/web-app-stalwart/files/playwright/test-oidc-provisioning.js new file mode 100644 index 0000000000..2aa8f9e472 --- /dev/null +++ b/roles/web-app-stalwart/files/playwright/test-oidc-provisioning.js @@ -0,0 +1,79 @@ +const { test, expect } = require("@playwright/test"); + +const { MAPACHE, safeSkipUnlessEnabled, gotoOnion } = require("./personas"); +const { isSplitRealmOidc, resolveTimeout } = require("./timeouts"); +const { roundcubeSsoLogin, roundcubeLogout, sendMail, waitForEmailInMailbox } = require("./webmail"); +const { + webmailBaseUrl, + mapachePassword, + biberEmail, + biberUsername, + biberPassword, +} = require("./env"); + +const mapacheUsername = MAPACHE.username; + +function mapacheLoginPassword() { + expect(mapachePassword, "MAPACHE_PASSWORD must be set").toBeTruthy(); + return mapachePassword; +} + +test("stalwart: an OIDC user the deploy never provisioned can sign in", async ({ page }) => { + test.skip(isSplitRealmOidc(), "clearnet app with an onion OIDC issuer: unreachable from one browser"); + safeSkipUnlessEnabled("sso"); + + expect(webmailBaseUrl, "WEBMAIL_BASE_URL must be set").toBeTruthy(); + + await roundcubeSsoLogin(page, mapacheUsername, mapacheLoginPassword()); + + await gotoOnion(page, `${webmailBaseUrl}/?_task=mail&_mbox=INBOX`); + await expect( + page.locator("#messagelist, .messagelist, [id*='messagelist']").first(), + "a mailbox the mail server created on demand must render for the OIDC user", + ).toBeVisible({ timeout: resolveTimeout(60_000) }); + + // An IMAP session opened against a missing account leaves Roundcube on its + // own error surface instead of the mail task, so assert we are not there. + await expect( + page.locator(".error, #message.error, .boxerror"), + "Roundcube must not surface an authentication or mailbox error", + ).toHaveCount(0); + + await roundcubeLogout(page); +}); + +// Signing in only proves the directory accepted the credentials. Delivery +// proves the account Stalwart created is a real mailbox with a routable +// address, which is what email_by_username has to get right. +test("stalwart: the auto-provisioned OIDC user sends mail that reaches biber", async ({ browser }) => { + test.skip(isSplitRealmOidc(), "clearnet app with an onion OIDC issuer: unreachable from one browser"); + safeSkipUnlessEnabled("sso"); + + expect(biberEmail, "BIBER_EMAIL must be set").toBeTruthy(); + + const subject = `mapache-to-biber-${Date.now()}`; + + const senderContext = await browser.newContext({ ignoreHTTPSErrors: true }); + const senderPage = await senderContext.newPage(); + try { + await roundcubeSsoLogin(senderPage, mapacheUsername, mapacheLoginPassword()); + await sendMail(senderPage, biberEmail, subject, "Sent by an account nobody provisioned."); + } finally { + await senderContext.close(); + } + + const recipientContext = await browser.newContext({ ignoreHTTPSErrors: true }); + const recipientPage = await recipientContext.newPage(); + try { + await roundcubeSsoLogin(recipientPage, biberUsername, biberPassword); + const row = await waitForEmailInMailbox( + recipientPage, + webmailBaseUrl, + subject, + resolveTimeout(90_000), + ); + await expect(row, `biber must receive ${subject} from ${mapacheUsername}`).toBeVisible(); + } finally { + await recipientContext.close(); + } +}); diff --git a/roles/web-app-stalwart/files/playwright/test-onion-mailflow.js b/roles/web-app-stalwart/files/playwright/test-onion-mailflow.js new file mode 100644 index 0000000000..dd5fae64d7 --- /dev/null +++ b/roles/web-app-stalwart/files/playwright/test-onion-mailflow.js @@ -0,0 +1,62 @@ +const { test, expect } = require("@playwright/test"); + +const { safeSkipUnlessEnabled, gotoOnion, recordVideoOptions } = require("./personas"); +const { roundcubeSsoLogin, roundcubeLogout } = require("./webmail"); +const { webmailBaseUrl, biberUsername, biberPassword } = require("./env"); +const { resolveTimeout, isSplitRealmOidc } = require("./timeouts"); + +// Outbound .onion delivery from the user's side: biber addresses a recipient on a +// foreign onion and the message leaves as queued mail instead of bouncing. +// +// The lab has no external onion mail peer, so this asserts the routing decision +// rather than a remote receipt: with MtaRoute + MtaOutboundStrategy in place the +// recipient is accepted and queued to the tor-smtp relay, whereas an unrouted +// .onion domain has no MX and returns a delivery failure to the sender. The +// recipient domain is deliberately not a local domain — is_local_domain wins over +// the .onion match in the strategy, so a self-addressed onion would route 'local' +// and never reach the gateway. +const FOREIGN_ONION = "recipient@nexusprobe7xk3mjqzvhbnd4rlyugc2pfe6sotai5w.onion"; + +test("biber: a .onion recipient is accepted and routed through the Tor gateway", async ({ browser }, testInfo) => { + test.skip(isSplitRealmOidc(), "clearnet app with an onion OIDC issuer: unreachable from one browser"); + safeSkipUnlessEnabled("tor"); + safeSkipUnlessEnabled("sso"); + expect(webmailBaseUrl, "WEBMAIL_BASE_URL must be set").toBeTruthy(); + expect(biberPassword, "BIBER_PASSWORD must be set").toBeTruthy(); + + const testSubject = `Playwright onion route ${Date.now()}`; + const proxyServer = process.env.PLAYWRIGHT_PROXY; + const context = await browser.newContext({ + ignoreHTTPSErrors: true, + ...(proxyServer ? { proxy: { server: proxyServer } } : {}), + ...recordVideoOptions(testInfo), + }); + + try { + const page = await context.newPage(); + await roundcubeSsoLogin(page, biberUsername, biberPassword); + await gotoOnion(page, `${webmailBaseUrl}/?_task=mail&_action=compose`); + await page.waitForLoadState("networkidle", { timeout: resolveTimeout(15_000) }).catch(() => {}); + await page.locator("#_to, input[name='_to']").first().fill(FOREIGN_ONION); + await page.locator("#compose-subject, input[name='_subject']").first().fill(testSubject); + await page.locator("#composebody, textarea[name='_message'], [contenteditable='true']").first() + .fill("Automated Playwright probe for the .onion outbound route."); + await page.locator(".formbuttons .send, button.send, a.send").first().click(); + + const sendError = page.locator("#messagestack .error, .toast .error, .toast-error").first(); + if (await sendError.isVisible().catch(() => false)) { + throw new Error(`Stalwart refused the .onion recipient: ${await sendError.textContent()}`); + } + + await gotoOnion(page, `${webmailBaseUrl}/?_task=mail&_mbox=INBOX`); + await page.waitForTimeout(resolveTimeout(10_000)); + await gotoOnion(page, `${webmailBaseUrl}/?_task=mail&_mbox=INBOX`); + await expect( + page.locator("#messagelist").getByText(/undelivered|delivery status|delivery failure|mailer-daemon/i).first() + ).toBeHidden({ timeout: resolveTimeout(15_000) }); + + await roundcubeLogout(page); + } finally { + await context.close().catch(() => {}); + } +}); diff --git a/roles/web-app-stalwart/files/playwright/test-personas.js b/roles/web-app-stalwart/files/playwright/test-personas.js new file mode 100644 index 0000000000..b7a1dc5049 --- /dev/null +++ b/roles/web-app-stalwart/files/playwright/test-personas.js @@ -0,0 +1,31 @@ +const { test, expect } = require("@playwright/test"); + +const { runAdminFlow, runBiberFlow, runGuestFlow } = require("./personas"); +const { resolveTimeout } = require("./timeouts"); + +// Shared persona flows (gated by PERSONA_*_BLOCKED in templates/playwright.env.j2). +test("guest: public-landing → auth chain → never authenticated", async ({ page }) => { + await runGuestFlow(page); +}); + +test("biber: app → universal logout", async ({ page }) => { + await runBiberFlow(page); +}); + +test("administrator: app → universal logout", async ({ page }) => { + await runAdminFlow(page, { + adminInteraction: async (interactivePage) => { + const link = interactivePage + .getByRole("link", { name: /^(domains|accounts|directory|settings|dashboard)$/i }) + .first(); + if (await link.isVisible().catch(() => false)) { + await link.click().catch(() => {}); + await interactivePage.waitForLoadState("domcontentloaded", { timeout: resolveTimeout(30_000) }).catch(() => {}); + await expect(interactivePage.locator("body")).toContainText( + /domains|accounts|directory|settings|dashboard/i, + { timeout: resolveTimeout(30_000) }, + ); + } + }, + }); +}); diff --git a/roles/web-app-stalwart/files/playwright/test-sso.js b/roles/web-app-stalwart/files/playwright/test-sso.js new file mode 100644 index 0000000000..cbe2cb7d98 --- /dev/null +++ b/roles/web-app-stalwart/files/playwright/test-sso.js @@ -0,0 +1,48 @@ +const { test, expect } = require("@playwright/test"); + +const { performKeycloakLoginForm, safeSkipUnlessEnabled, gotoOnion } = require("./personas"); +const { isSplitRealmOidc } = require("./timeouts"); +const { + appBaseUrl, + canonicalDomain, + expectedOidcAuthUrl, + adminEmail, + adminUsername, + adminPassword, +} = require("./env"); +const { resolveTimeout } = require("./timeouts"); + +// WebAdmin SSO: a username-first OAuth flow — enter the email, and (the mail +// domain being bound to our OIDC directory) the webui redirects to Keycloak. +test("stalwart: sso login, open WebAdmin, logout", async ({ page }) => { + test.skip(isSplitRealmOidc(), "clearnet app with an onion OIDC issuer: unreachable from one browser"); + safeSkipUnlessEnabled("sso"); + + await gotoOnion(page, `${appBaseUrl}/`); + await expect.poll(() => page.url(), { timeout: resolveTimeout(30_000) }).toContain(`${canonicalDomain}/account/login`); + + const loginField = page + .getByRole("textbox", { name: /email|user|login/i }) + .or(page.locator("input[type='email'], input[type='text'], input[name='username'], input[name='email']")) + .first(); + await loginField.waitFor({ state: "visible", timeout: resolveTimeout(30_000) }); + await loginField.fill(adminEmail); + await page + .getByRole("button", { name: /continue|next|log ?in|sign ?in/i }) + .or(page.locator("button[type='submit']")) + .first() + .click(); + + await expect.poll(() => page.url(), { timeout: resolveTimeout(60_000) }).toContain(expectedOidcAuthUrl); + await performKeycloakLoginForm(page, adminUsername, adminPassword); + await expect.poll(() => page.url(), { timeout: resolveTimeout(60_000) }).toContain(canonicalDomain); + + await expect( + page.locator("nav, .sidebar, [class*='menu'], h1, h2").filter({ hasText: /dashboard|domains|account|settings|directory/i }).first() + ).toBeVisible({ timeout: resolveTimeout(30_000) }); + + const logout = page.locator("a[href*='logout'], button:has-text('Logout')").or(page.getByRole("link", { name: /logout/i })); + if (await logout.first().isVisible().catch(() => false)) { + await logout.first().click(); + } +}); diff --git a/roles/web-app-stalwart/files/playwright/test-tls.js b/roles/web-app-stalwart/files/playwright/test-tls.js new file mode 100644 index 0000000000..372247a8c9 --- /dev/null +++ b/roles/web-app-stalwart/files/playwright/test-tls.js @@ -0,0 +1,28 @@ +const { test, expect } = require("@playwright/test"); + +const { appBaseUrl, canonicalDomain } = require("./env"); +const { gotoOnion } = require("./personas"); +const { resolveTimeout, isOnionTarget } = require("./timeouts"); + +// Baseline reachability: WebAdmin TLS + DAV auto-discovery through the proxy. +test("stalwart: WebAdmin is served under canonical domain with TLS", async ({ page }) => { + const response = await gotoOnion(page, `${appBaseUrl}/`); + expect(response, "Expected Stalwart response").toBeTruthy(); + expect(response.status(), "Expected status < 400").toBeLessThan(400); + expect(response.url().includes(canonicalDomain), `Expected canonical domain "${canonicalDomain}"`).toBe(true); + if (!isOnionTarget()) { + expect(response.headers()["strict-transport-security"], "Stalwart must emit HSTS").toBeTruthy(); + } +}); + +test("stalwart: CalDAV/CardDAV discovery is reachable", async ({ request }) => { + for (const [wk, path] of [["caldav", "/dav/cal"], ["carddav", "/dav/card"]]) { + const res = await request.get(`${appBaseUrl}/.well-known/${wk}`, { + maxRedirects: 0, + timeout: resolveTimeout(30_000), + }); + expect([301, 302, 307, 308].includes(res.status()), + `${wk} well-known must redirect (got ${res.status()})`).toBe(true); + expect(res.headers()["location"] || "", `${wk} should point at ${path}`).toContain("/dav/"); + } +}); diff --git a/roles/web-app-stalwart/files/playwright/webmail.js b/roles/web-app-stalwart/files/playwright/webmail.js new file mode 100644 index 0000000000..5420e44837 --- /dev/null +++ b/roles/web-app-stalwart/files/playwright/webmail.js @@ -0,0 +1,49 @@ +const { expect } = require("@playwright/test"); + +const { performKeycloakLoginForm, gotoOnion, waitForEmailInMailbox } = require("./personas"); +const { webmailBaseUrl, expectedOidcAuthUrl } = require("./env"); +const { resolveTimeout } = require("./timeouts"); + +async function roundcubeSsoLogin(page, username, password) { + await gotoOnion(page, `${webmailBaseUrl}/`); + const ssoButton = page + .getByRole("button", { name: /sso|single sign.?on|login with|openid/i }) + .or(page.getByRole("link", { name: /sso|single sign.?on|login with|openid/i })); + if (await ssoButton.first().isVisible().catch(() => false)) { + await ssoButton.first().click(); + } + await expect + .poll(() => page.url(), { + timeout: resolveTimeout(30_000), + message: `Expected redirect to Keycloak OIDC: ${expectedOidcAuthUrl}`, + }) + .toContain(expectedOidcAuthUrl); + await performKeycloakLoginForm(page, username, password); + await page.locator("#messagelist, .compose, a[href*='_action=compose'], .toolbar").first() + .waitFor({ state: "visible", timeout: resolveTimeout(60_000) }); +} + +/** Compose and send one message, failing loudly if Roundcube reports a send error. */ +async function sendMail(page, recipient, subject, body) { + await gotoOnion(page, `${webmailBaseUrl}/?_task=mail&_action=compose`); + await page.waitForLoadState("networkidle", { timeout: resolveTimeout(15_000) }).catch(() => {}); + await page.locator("#_to, input[name='_to']").first().fill(recipient); + await page.locator("#compose-subject, input[name='_subject']").first().fill(subject); + await page.locator("#composebody, textarea[name='_message'], [contenteditable='true']").first().fill(body); + await page.locator(".formbuttons .send, button.send, a.send").first().click(); + const sendError = page.locator("#messagestack .error, .toast .error, .toast-error").first(); + if (await sendError.isVisible().catch(() => false)) { + throw new Error(`Roundcube reported a send error for ${recipient}: ${await sendError.textContent()}`); + } +} + +async function roundcubeLogout(page) { + const logout = page.locator("a[href*='_task=logout'], a[href*='logout'], a.logout") + .or(page.getByRole("link", { name: /logout|sign out/i })); + await expect(logout.first(), "Roundcube logout control must be present").toBeVisible({ + timeout: resolveTimeout(10_000), + }); + await logout.first().click(); +} + +module.exports = { roundcubeSsoLogin, roundcubeLogout, sendMail, waitForEmailInMailbox }; diff --git a/roles/web-app-stalwart/files/python/migrate_from_mailu.py b/roles/web-app-stalwart/files/python/migrate_from_mailu.py new file mode 100755 index 0000000000..a76b117ed5 --- /dev/null +++ b/roles/web-app-stalwart/files/python/migrate_from_mailu.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Migrate mailbox data from a Mailu installation into Stalwart. + +Mailu stores mail as standard Dovecot Maildir under its ``dovecot_mail`` +volume (one ``/`` tree per account, subfolders as ``.Name`` maildir +folders). Reading that volume directly means NO live Mailu is required — +the migration works against a stopped or legacy-parked instance and the +test harness needs no full platform deploy. + +Scope (v1, deliberate): + - Migrates MESSAGES (folder structure, flags, internal dates) per account + via IMAP APPEND into Stalwart. + - Idempotent: a message whose Message-ID already exists in the target + folder is skipped, so re-runs converge instead of duplicating. + - Accounts/aliases are NOT created here: on the platform they are + provisioned from the inventory (web-app-stalwart tasks/04_manage_user.yml); + the test harness creates its fixture account via the same JMAP API. + - Sieve filters and CalDAV/CardDAV data are out of scope (documented in + the role README). + +Usage: + migrate_from_mailu.py --maildir-root /mail --imap-host mail.example.org \ + --accounts-file accounts.json [--imap-port 993] [--imap-insecure] \ + [--dest-separator /] + +``accounts.json``: {"": "", ...} — the credentials of +the already-provisioned Stalwart accounts (inventory passwords on the +platform; fixture passwords in the test). +""" + +from __future__ import annotations + +import argparse +import contextlib +import email.parser +import imaplib +import json +import ssl +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path + +_MAILDIR_FLAG_MAP = { + "S": "\\Seen", + "R": "\\Answered", + "F": "\\Flagged", + "T": "\\Deleted", + "D": "\\Draft", +} + + +@dataclass +class FolderStats: + appended: int = 0 + skipped: int = 0 + failed: int = 0 + + +@dataclass +class AccountStats: + folders: dict[str, FolderStats] = field(default_factory=dict) + + def folder(self, name: str) -> FolderStats: + return self.folders.setdefault(name, FolderStats()) + + @property + def failed(self) -> int: + return sum(f.failed for f in self.folders.values()) + + +def maildir_flags_to_imap(filename: str) -> str: + """Map a maildir file's ``:2,`` suffix to an IMAP flag list string.""" + if ":2," not in filename: + return "" + info = filename.rsplit(":2,", 1)[1] + flags = [_MAILDIR_FLAG_MAP[c] for c in info if c in _MAILDIR_FLAG_MAP] + return "(" + " ".join(flags) + ")" if flags else "" + + +def maildir_folder_to_imap(maildir_name: str, separator: str) -> str: + """Map a maildir folder dir name to the destination IMAP mailbox name. + + ``""`` (the maildir root) is INBOX; ``.Archive.2024`` becomes + ``Archive2024`` (dovecot uses ``.`` as the on-disk separator). + """ + if not maildir_name: + return "INBOX" + return maildir_name.lstrip(".").replace(".", separator) + + +def iter_maildir_folders(account_dir: Path) -> list[tuple[str, Path]]: + """Yield (maildir_folder_name, path) for the root and every ``.Sub`` dir.""" + folders: list[tuple[str, Path]] = [] + if (account_dir / "cur").is_dir() or (account_dir / "new").is_dir(): + folders.append(("", account_dir)) + folders.extend( + (child.name, child) + for child in sorted(account_dir.iterdir()) + if child.is_dir() + and child.name.startswith(".") + and ((child / "cur").is_dir() or (child / "new").is_dir()) + ) + return folders + + +def iter_messages(folder_dir: Path) -> list[Path]: + """All message files of one maildir folder (cur = seen-ish, new = unseen).""" + messages: list[Path] = [] + for sub in ("cur", "new"): + subdir = folder_dir / sub + if subdir.is_dir(): + messages.extend(p for p in sorted(subdir.iterdir()) if p.is_file()) + return messages + + +def extract_message_id(raw: bytes) -> str: + """Message-ID header value from raw RFC-5322 bytes ('' when absent).""" + parser = email.parser.BytesHeaderParser() + try: + return (parser.parsebytes(raw).get("Message-ID") or "").strip() + except Exception: + return "" + + +_FETCH_CHUNK = 500 + + +def existing_message_ids(imap: imaplib.IMAP4) -> set[str]: + """Message-IDs already present in the SELECTed folder. + + Fetched client-side (one header-fields FETCH per chunk) instead of + ``SEARCH HEADER``: Stalwart indexes headers asynchronously, so a + server-side search right after APPEND misses fresh messages — the + harness's idempotency step caught exactly that as duplicates. + """ + typ, nums = imap.search(None, "ALL") + if typ != "OK" or not nums or not nums[0].split(): + return set() + sequence = [n.decode() for n in nums[0].split()] + ids: set[str] = set() + for start in range(0, len(sequence), _FETCH_CHUNK): + chunk = ",".join(sequence[start : start + _FETCH_CHUNK]) + typ, data = imap.fetch(chunk, "(BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])") + if typ != "OK": + continue + for part in data or []: + if isinstance(part, tuple): + message_id = extract_message_id(part[1]) + if message_id: + ids.add(message_id) + return ids + + +def ensure_mailbox(imap: imaplib.IMAP4, mailbox: str) -> None: + """CREATE the mailbox; 'already exists' answers are fine.""" + if mailbox.upper() == "INBOX": + return + imap.create(mailbox) + + +def migrate_account( + imap: imaplib.IMAP4, + account_dir: Path, + separator: str, + stats: AccountStats, +) -> None: + for maildir_name, folder_dir in iter_maildir_folders(account_dir): + mailbox = maildir_folder_to_imap(maildir_name, separator) + fstats = stats.folder(mailbox) + ensure_mailbox(imap, mailbox) + typ, _ = imap.select(mailbox) + if typ != "OK": + print(f" [FAIL] cannot SELECT {mailbox!r}", file=sys.stderr) + fstats.failed += len(iter_messages(folder_dir)) + continue + present = existing_message_ids(imap) + for msg_path in iter_messages(folder_dir): + raw = msg_path.read_bytes() + message_id = extract_message_id(raw) + if message_id and message_id in present: + fstats.skipped += 1 + continue + present.add(message_id) + flags = maildir_flags_to_imap(msg_path.name) + internal_date = imaplib.Time2Internaldate(msg_path.stat().st_mtime) + typ, data = imap.append(mailbox, flags or None, internal_date, raw) + if typ == "OK": + fstats.appended += 1 + else: + print(f" [FAIL] APPEND {msg_path.name}: {data}", file=sys.stderr) + fstats.failed += 1 + + +def connect(host: str, port: int, insecure: bool) -> imaplib.IMAP4_SSL: + context = ssl.create_default_context() + if insecure: + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + imap = imaplib.IMAP4_SSL(host, port, ssl_context=context, timeout=30) + imap._encoding = "utf-8" + return imap + + +def run(args: argparse.Namespace) -> int: + accounts: dict[str, str] = json.loads( + Path(args.accounts_file).read_text(encoding="utf-8") + ) + maildir_root = Path(args.maildir_root) + exit_code = 0 + + for address, password in accounts.items(): + account_dir = maildir_root / address + if not account_dir.is_dir(): + print(f"[SKIP] no maildir for {address} under {maildir_root}") + continue + print(f"[ACCOUNT] {address}") + stats = AccountStats() + imap = connect(args.imap_host, args.imap_port, args.imap_insecure) + try: + imap.login(address, password) + migrate_account(imap, account_dir, args.dest_separator, stats) + finally: + with contextlib.suppress(Exception): + imap.logout() + for mailbox, f in sorted(stats.folders.items()): + print( + f" {mailbox}: appended={f.appended} " + f"skipped={f.skipped} failed={f.failed}" + ) + if stats.failed: + exit_code = 1 + return exit_code + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--maildir-root", + required=True, + help="Mailu dovecot mail volume root (contains one / tree per account)", + ) + parser.add_argument("--imap-host", required=True, help="Stalwart IMAP host") + parser.add_argument("--imap-port", type=int, default=993) + parser.add_argument( + "--imap-insecure", + action="store_true", + help="Skip TLS verification (fixture/bootstrap certificates)", + ) + parser.add_argument( + "--accounts-file", + required=True, + help='JSON file {"": ""} of provisioned Stalwart accounts', + ) + parser.add_argument( + "--dest-separator", + default="/", + help="IMAP hierarchy separator on the destination (Stalwart: '/')", + ) + return parser.parse_args(argv) + + +if __name__ == "__main__": + start = time.time() + rc = run(parse_args()) + print(f"[DONE] rc={rc} in {time.time() - start:.1f}s") + raise SystemExit(rc) diff --git a/roles/web-app-stalwart/files/python/production_facts.py b/roles/web-app-stalwart/files/python/production_facts.py new file mode 100755 index 0000000000..dab5f17ab6 --- /dev/null +++ b/roles/web-app-stalwart/files/python/production_facts.py @@ -0,0 +1,557 @@ +#!/usr/bin/env python3 +"""Assert the public mail facts of a deployed Stalwart server. + +Read-only: this queries DNS and opens one TCP connection to the SMTP port, so a +misconfigured zone fails the deploy instead of silently degrading deliverability. +Only hard facts are checked -- records, ports and banners that either match the +deployed configuration or do not; mail flow is covered by the role's other suites. + +A minimal DNS client lives in this file on purpose: the project ships no DNS +library, `dig` is not guaranteed on a target host, and the checks must behave +identically in DiD, in CI and on a production server. + +# nocheck: file-size — the client and the assertions cannot be split: this runs +# through `ansible.builtin.script`, which ships exactly one file to the target. +""" + +from __future__ import annotations + +import argparse +import secrets +import socket +import struct +import sys +import time +from pathlib import Path + +TYPE_A = 1 +TYPE_CNAME = 5 +TYPE_PTR = 12 +TYPE_MX = 15 +TYPE_TXT = 16 +TYPE_AAAA = 28 +TYPE_SRV = 33 + +CLASS_IN = 1 +RCODE_NXDOMAIN = 3 +_MAX_POINTER_HOPS = 64 +_UDP_SIZE = 4096 + + +class DnsError(RuntimeError): + """A DNS query could not be completed against any configured resolver.""" + + +def encode_name(name: str) -> bytes: + """Encode a domain name into DNS wire format (length-prefixed labels).""" + out = bytearray() + for label in name.rstrip(".").split("."): + if not label: + continue + raw = label.encode() if label.isascii() else label.encode("idna") + out.append(len(raw)) + out.extend(raw) + out.append(0) + return bytes(out) + + +def decode_name(buf: bytes, offset: int) -> tuple[str, int]: + """Decode a possibly compressed name; returns (name, offset after the name). + + The returned offset is the position after the name *as encoded at* ``offset`` + -- following a compression pointer must not advance the caller's cursor. + """ + labels: list[str] = [] + jumped = False + cursor = offset + after = offset + for _ in range(_MAX_POINTER_HOPS): + if cursor >= len(buf): + raise DnsError("truncated name in DNS response") + length = buf[cursor] + if length == 0: + cursor += 1 + if not jumped: + after = cursor + return ".".join(labels), after + if length & 0xC0 == 0xC0: + if cursor + 1 >= len(buf): + raise DnsError("truncated compression pointer in DNS response") + pointer = ((length & 0x3F) << 8) | buf[cursor + 1] + if not jumped: + after = cursor + 2 + jumped = True + cursor = pointer + continue + start = cursor + 1 + end = start + length + if end > len(buf): + raise DnsError("truncated label in DNS response") + labels.append(buf[start:end].decode("utf-8", "replace")) + cursor = end + raise DnsError("compression pointer loop in DNS response") + + +def build_query(name: str, rtype: int, txid: int) -> bytes: + """Build a recursion-desired query packet for one name/type.""" + header = struct.pack("!HHHHHH", txid, 0x0100, 1, 0, 0, 0) + return header + encode_name(name) + struct.pack("!HH", rtype, CLASS_IN) + + +def _decode_rdata(rtype: int, buf: bytes, offset: int, rdlength: int) -> object: + if rtype in (TYPE_A, TYPE_AAAA): + family = socket.AF_INET if rtype == TYPE_A else socket.AF_INET6 + return socket.inet_ntop(family, buf[offset : offset + rdlength]) + if rtype == TYPE_TXT: + parts: list[str] = [] + cursor = offset + end = offset + rdlength + while cursor < end: + chunk = buf[cursor] + cursor += 1 + parts.append(buf[cursor : cursor + chunk].decode("utf-8", "replace")) + cursor += chunk + return "".join(parts) + if rtype == TYPE_MX: + preference = struct.unpack("!H", buf[offset : offset + 2])[0] + target, _ = decode_name(buf, offset + 2) + return (preference, target) + if rtype == TYPE_SRV: + priority, weight, port = struct.unpack("!HHH", buf[offset : offset + 6]) + target, _ = decode_name(buf, offset + 6) + return (priority, weight, port, target) + if rtype in (TYPE_PTR, TYPE_CNAME): + return decode_name(buf, offset)[0] + return buf[offset : offset + rdlength] + + +def parse_response(buf: bytes, txid: int, rtype: int) -> list[object]: + """Return the decoded answer records of the requested type.""" + if len(buf) < 12: + raise DnsError("short DNS response") + resp_id, flags, qdcount, ancount, _ns, _ar = struct.unpack("!HHHHHH", buf[:12]) + if resp_id != txid: + raise DnsError("DNS transaction id mismatch") + rcode = flags & 0x000F + if rcode == RCODE_NXDOMAIN: + return [] + if rcode != 0: + raise DnsError(f"DNS server returned rcode {rcode}") + + offset = 12 + for _ in range(qdcount): + _, offset = decode_name(buf, offset) + offset += 4 + + answers: list[object] = [] + for _ in range(ancount): + _, offset = decode_name(buf, offset) + atype, _aclass, _ttl, rdlength = struct.unpack( + "!HHIH", buf[offset : offset + 10] + ) + offset += 10 + if atype == rtype: + answers.append(_decode_rdata(atype, buf, offset, rdlength)) + offset += rdlength + return answers + + +def _is_truncated(buf: bytes) -> bool: + if len(buf) < 12: + return False + return bool((struct.unpack("!H", buf[2:4])[0] >> 9) & 1) + + +def _query_one(resolver: str, packet: bytes, timeout: float) -> bytes: + family = socket.getaddrinfo(resolver, 53, type=socket.SOCK_DGRAM)[0][0] + with socket.socket(family, socket.SOCK_DGRAM) as sock: + sock.settimeout(timeout) + sock.sendto(packet, (resolver, 53)) + return sock.recv(_UDP_SIZE) + + +def _query_one_tcp(resolver: str, packet: bytes, timeout: float) -> bytes: + family = socket.getaddrinfo(resolver, 53, type=socket.SOCK_STREAM)[0][0] + with socket.socket(family, socket.SOCK_STREAM) as sock: + sock.settimeout(timeout) + sock.connect((resolver, 53)) + sock.sendall(struct.pack("!H", len(packet)) + packet) + header = _recv_exact(sock, 2) + return _recv_exact(sock, struct.unpack("!H", header)[0]) + + +def _recv_exact(sock: socket.socket, count: int) -> bytes: + chunks: list[bytes] = [] + remaining = count + while remaining > 0: + chunk = sock.recv(remaining) + if not chunk: + raise DnsError("connection closed mid-response") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def system_resolvers() -> list[str]: + """Nameserver addresses from /etc/resolv.conf, in file order.""" + try: + lines = Path("/etc/resolv.conf").read_text(encoding="utf-8").splitlines() + except OSError: + return [] + found: list[str] = [] + for raw in lines: + line = raw.strip() + if line.startswith("nameserver"): + parts = line.split() + if len(parts) >= 2: + found.append(parts[1]) + return found + + +def query(name: str, rtype: int, resolvers: list[str], timeout: float) -> list[object]: + """Query every resolver in turn; the first usable answer wins.""" + errors: list[str] = [] + for resolver in resolvers: + txid = secrets.randbelow(65536) + packet = build_query(name, rtype, txid) + try: + raw = _query_one(resolver, packet, timeout) + if _is_truncated(raw): + raw = _query_one_tcp(resolver, packet, timeout) + except (OSError, DnsError, struct.error, IndexError) as exc: + errors.append(f"{resolver}: {exc}") + continue + try: + answers = parse_response(raw, txid, rtype) + except (DnsError, struct.error, IndexError) as exc: + errors.append(f"{resolver}: {exc}") + else: + return answers + raise DnsError( + f"no resolver answered for {name}: {'; '.join(errors) or 'none configured'}" + ) + + +class Report: + """Collects pass/warn/fail lines and decides the exit code.""" + + def __init__(self) -> None: + self.failures = 0 + self.warnings = 0 + self.checks: dict[str, int] = {} + + def record(self, name: str, failures_before: int) -> None: + """Mark check *name* passed when it added no failure since ``failures_before``.""" + self.checks[name] = int(self.failures == failures_before) + + def ok(self, message: str) -> None: + print(f"OK: {message}") + + def warn(self, message: str) -> None: + self.warnings += 1 + print(f"WARN: {message}") + + def fail(self, message: str) -> None: + self.failures += 1 + print(f"FAIL: {message}", file=sys.stderr) + + +def _resolve_addresses(host: str, resolvers: list[str], timeout: float) -> list[str]: + addresses = [str(a) for a in query(host, TYPE_A, resolvers, timeout)] + addresses.extend(str(a) for a in query(host, TYPE_AAAA, resolvers, timeout)) + return addresses + + +def check_host_address(args, resolvers, report: Report) -> list[str]: + """The mail host resolves to at least one address.""" + addresses = _resolve_addresses(args.mail_host, resolvers, args.timeout) + if not addresses: + report.fail(f"{args.mail_host} has no A/AAAA record") + return [] + report.ok(f"{args.mail_host} resolves to {', '.join(addresses)}") + return addresses + + +def check_mx(args, resolvers, report: Report) -> None: + """The zone has exactly one MX and it is the mail host (role writes solo).""" + records = query(args.mail_domain, TYPE_MX, resolvers, args.timeout) + targets = [str(target).rstrip(".").lower() for _pref, target in records] + if not targets: + report.fail(f"{args.mail_domain} has no MX record") + return + expected = args.mail_host.rstrip(".").lower() + if expected not in targets: + report.fail(f"MX of {args.mail_domain} is {targets}, expected {expected}") + return + if len(targets) > 1: + report.fail(f"MX of {args.mail_domain} is not solo: {targets}") + return + report.ok(f"MX of {args.mail_domain} -> {expected}") + + +def check_spf(args, resolvers, report: Report) -> None: + """A single v=spf1 record exists and authorises the mail host.""" + records = [ + str(r) for r in query(args.mail_domain, TYPE_TXT, resolvers, args.timeout) + ] + spf = [r for r in records if r.lower().startswith("v=spf1")] + if not spf: + report.fail(f"{args.mail_domain} has no v=spf1 TXT record") + return + if len(spf) > 1: + report.fail( + f"{args.mail_domain} has {len(spf)} SPF records; exactly one is valid" + ) + return + value = spf[0] + host = args.mail_host.rstrip(".").lower() + if f"a:{host}" not in value.lower() and " mx" not in f" {value.lower()}": + report.fail( + f"SPF of {args.mail_domain} authorises neither 'mx' nor 'a:{host}': {value}" + ) + return + report.ok(f"SPF of {args.mail_domain}: {value}") + + +def check_dmarc(args, resolvers, report: Report) -> None: + """_dmarc TXT exists, is a DMARC1 record and declares a policy.""" + name = f"_dmarc.{args.mail_domain}" + records = [str(r) for r in query(name, TYPE_TXT, resolvers, args.timeout)] + dmarc = [r for r in records if r.lower().startswith("v=dmarc1")] + if not dmarc: + report.fail(f"{name} has no v=DMARC1 TXT record") + return + if "p=" not in dmarc[0].lower(): + report.fail(f"{name} declares no policy (p=): {dmarc[0]}") + return + report.ok(f"DMARC of {args.mail_domain}: {dmarc[0]}") + + +def check_dkim(args, resolvers, report: Report) -> None: + """The deployed selector publishes a DKIM1 record with a non-empty key.""" + if not args.dkim_selector: + report.warn("no DKIM selector supplied; skipping the DKIM record check") + return + name = f"{args.dkim_selector}._domainkey.{args.mail_domain}" + records = [str(r) for r in query(name, TYPE_TXT, resolvers, args.timeout)] + dkim = [r for r in records if "v=dkim1" in r.lower()] + if not dkim: + report.fail(f"{name} has no v=DKIM1 TXT record") + return + key = "" + for field in dkim[0].split(";"): + stripped = field.strip() + if stripped.lower().startswith("p="): + key = stripped[2:].strip() + if not key: + report.fail(f"{name} publishes an empty DKIM key (p=): {dkim[0]}") + return + report.ok(f"DKIM {args.dkim_selector} of {args.mail_domain}: {len(key)}-char key") + + +def check_srv(args, resolvers, report: Report) -> None: + """Every advertised SRV points at the mail host on its implicit-TLS port.""" + for service, port in (("_submissions", 465), ("_imaps", 993), ("_pop3s", 995)): + name = f"{service}._tcp.{args.mail_domain}" + records = query(name, TYPE_SRV, resolvers, args.timeout) + if not records: + report.fail(f"{name} has no SRV record") + continue + matches = [ + rec + for rec in records + if rec[2] == port + and str(rec[3]).rstrip(".").lower() == args.mail_host.rstrip(".").lower() + ] + if not matches: + report.fail( + f"{name} does not point at {args.mail_host}:{port} (got {records})" + ) + continue + report.ok(f"{name} -> {args.mail_host}:{port}") + + +def check_ptr(args, addresses: list[str], resolvers, report: Report) -> None: + """PTR must not contradict the mail host; an absent PTR is only a warning. + + A wrong PTR is a deliverability bug this deploy can cause. A missing one is + usually outside the role's reach -- rDNS is only provisioned for Hetzner. + """ + expected = args.mail_host.rstrip(".").lower() + for address in addresses: + if ":" in address: + continue + reverse = ".".join(reversed(address.split("."))) + ".in-addr.arpa" + names = [ + str(n).rstrip(".").lower() + for n in query(reverse, TYPE_PTR, resolvers, args.timeout) + ] + if not names: + report.warn( + f"{address} has no PTR record (rDNS is provisioned for Hetzner only)" + ) + continue + if expected not in names: + report.fail(f"PTR of {address} is {names}, expected {expected}") + continue + report.ok(f"PTR of {address} -> {expected}") + + +def check_smtp_banner(args, report: Report) -> None: + """Port 25 answers with a 220 greeting carrying the mail host FQDN.""" + try: + with socket.create_connection( + (args.mail_host, args.smtp_port), timeout=args.timeout + ) as sock: + sock.settimeout(args.timeout) + banner = sock.recv(512).decode("utf-8", "replace").strip() + except OSError as exc: + report.fail(f"{args.mail_host}:{args.smtp_port} is not reachable: {exc}") + return + if not banner.startswith("220"): + report.fail( + f"{args.mail_host}:{args.smtp_port} greeting is not 220: {banner!r}" + ) + return + if args.mail_host.rstrip(".").lower() not in banner.lower(): + report.fail( + f"SMTP greeting does not carry {args.mail_host} (rejectNonFqdn identity): {banner!r}" + ) + return + report.ok(f"SMTP greeting on {args.mail_host}:{args.smtp_port}: {banner}") + + +def run_checks(args, resolvers: list[str]) -> Report: + """Run every hard-fact check once and return the collected report.""" + report = Report() + + def run(name: str, fn, *fn_args): + before = report.failures + result = fn(*fn_args) + report.record(name, before) + return result + + addresses = run("host_address", check_host_address, args, resolvers, report) + run("mx", check_mx, args, resolvers, report) + run("spf", check_spf, args, resolvers, report) + run("dmarc", check_dmarc, args, resolvers, report) + run("dkim", check_dkim, args, resolvers, report) + run("srv", check_srv, args, resolvers, report) + if addresses: + run("ptr", check_ptr, args, addresses, resolvers, report) + run("smtp_banner", check_smtp_banner, args, report) + return report + + +def _label(value: str) -> str: + return value.replace("\\", "\\\\").replace('"', '\\"') + + +def write_metrics(report: Report, args, path: str) -> None: + """Render *report* as a Prometheus textfile at *path*. + + Written to a sibling temp file and renamed, because node_exporter's + textfile collector reads whole files and would otherwise pick up a + half-written scrape. + """ + scope = f'host="{_label(args.mail_host)}",domain="{_label(args.mail_domain)}"' + lines = [ + "# HELP stalwart_mail_fact_up Whether one production mail configuration check passed.", + "# TYPE stalwart_mail_fact_up gauge", + ] + for name, passed in sorted(report.checks.items()): + lines.append( + f'stalwart_mail_fact_up{{{scope},check="{_label(name)}"}} {passed}' + ) + lines += [ + "# HELP stalwart_mail_facts_failures Failing production mail configuration checks.", + "# TYPE stalwart_mail_facts_failures gauge", + f"stalwart_mail_facts_failures{{{scope}}} {report.failures}", + "# HELP stalwart_mail_facts_warnings Non-fatal findings from the same run.", + "# TYPE stalwart_mail_facts_warnings gauge", + f"stalwart_mail_facts_warnings{{{scope}}} {report.warnings}", + "# HELP stalwart_mail_facts_last_run_timestamp_seconds When the checks last completed.", + "# TYPE stalwart_mail_facts_last_run_timestamp_seconds gauge", + f"stalwart_mail_facts_last_run_timestamp_seconds{{{scope}}} {int(time.time())}", + ] + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + staged = target.with_suffix(f"{target.suffix}.{secrets.token_hex(4)}") + staged.write_text("\n".join(lines) + "\n", encoding="utf-8") + staged.replace(target) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--mail-host", required=True, help="Public FQDN of the mail server" + ) + parser.add_argument( + "--mail-domain", required=True, help="Zone carrying MX/SPF/DMARC" + ) + parser.add_argument( + "--dkim-selector", default="", help="Selector published by the server" + ) + parser.add_argument("--smtp-port", type=int, default=25) + parser.add_argument("--timeout", type=float, default=5.0) + parser.add_argument( + "--resolver", + action="append", + default=[], + help="Resolver to query (repeatable); defaults to /etc/resolv.conf", + ) + parser.add_argument( + "--retries", + type=int, + default=6, + help="Whole-suite retries; absorbs DNS propagation right after a deploy", + ) + parser.add_argument("--retry-delay", type=float, default=10.0) + parser.add_argument( + "--metrics-file", + default="", + help="Write the outcome as a Prometheus textfile for node_exporter to expose", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + resolvers = args.resolver or system_resolvers() + if not resolvers: + print("FAIL: no resolver configured; pass --resolver", file=sys.stderr) + return 1 + + print( + f"Checking {args.mail_host} (zone {args.mail_domain}) via {', '.join(resolvers)}" + ) + report = Report() + for attempt in range(1, args.retries + 1): + try: + report = run_checks(args, resolvers) + except DnsError as exc: + report = Report() + report.fail(str(exc)) + if report.failures == 0: + break + if attempt < args.retries: + print( + f"--- {report.failures} failure(s); retry {attempt}/{args.retries - 1} ---" + ) + time.sleep(args.retry_delay) + + if args.metrics_file: + write_metrics(report, args, args.metrics_file) + + if report.failures: + print( + f"Production mail facts FAILED: {report.failures} failure(s).", + file=sys.stderr, + ) + return 1 + print(f"Production mail facts passed ({report.warnings} warning(s)).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/roles/web-app-stalwart/files/test/.nocheck b/roles/web-app-stalwart/files/test/.nocheck new file mode 100644 index 0000000000..9d0e813947 --- /dev/null +++ b/roles/web-app-stalwart/files/test/.nocheck @@ -0,0 +1 @@ +# nocheck: role-files-layout - Role test harness with test.sh as its entry point; the e2e runner stages the tree by path and invokes test.sh from it. diff --git a/roles/web-app-stalwart/files/test/probe_mail.py b/roles/web-app-stalwart/files/test/probe_mail.py new file mode 100755 index 0000000000..b6210e7f4d --- /dev/null +++ b/roles/web-app-stalwart/files/test/probe_mail.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Send one message, or look one up, without depending on curl's optional protocols. + +`curl` is built per distro and its SMTP and IMAP protocol handlers are optional: +on CentOS the system curl reports `Protocol "smtp" not supported`, so a harness +that shells out to it passes on Debian and fails on CentOS for reasons that have +nothing to do with the mail server. Python's stdlib carries both protocols on +every distro the matrix runs, and the migration script it exercises already +speaks IMAP through imaplib. + + probe_mail.py send + probe_mail.py find + +`find` exits 0 when the subject is present in INBOX or Junk, 1 when it is not. +Both mailboxes are searched because a test domain publishes no mail-auth DNS, so +a correctly delivered message is legitimately filed under Junk. Certificate +verification is off for the same reason the migration script turns it off: the +loopback hop reaches Stalwart's bootstrap certificate, which matches no hostname. +`imaplib` is forced to UTF-8 for the same reason too — it encodes command +arguments as ASCII, so a password holding any non-ASCII character (the platform +generates passwords containing '€') raises UnicodeEncodeError before LOGIN is +ever sent. +""" + +from __future__ import annotations + +import contextlib +import imaplib +import smtplib +import ssl +import sys +from pathlib import Path + +SMTP_PORT = 25 +IMAPS_PORT = 993 +TIMEOUT = 60 + +MAILBOXES = ("INBOX", "Junk Mail") + + +def send(host: str, sender: str, recipient: str, eml_path: str) -> int: + message = Path(eml_path).read_bytes() + with smtplib.SMTP(host, SMTP_PORT, timeout=TIMEOUT) as smtp: + smtp.sendmail(sender, [recipient], message) + return 0 + + +def find(host: str, user: str, password: str, subject: str) -> int: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + imap = imaplib.IMAP4_SSL(host, IMAPS_PORT, ssl_context=context, timeout=TIMEOUT) + imap._encoding = "utf-8" + try: + imap.login(user, password) + for mailbox in MAILBOXES: + typ, _ = imap.select(f'"{mailbox}"', readonly=True) + if typ != "OK": + continue + typ, data = imap.search(None, "SUBJECT", f'"{subject}"') + if typ == "OK" and data and data[0].split(): + return 0 + finally: + with contextlib.suppress(Exception): + imap.logout() + return 1 + + +def main(argv: list[str]) -> int: + if len(argv) < 2: + print(__doc__, file=sys.stderr) + return 2 + action, args = argv[1], argv[2:] + if action == "send" and len(args) == 4: + return send(*args) + if action == "find" and len(args) == 4: + return find(*args) + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/roles/web-app-stalwart/files/test/test.sh b/roles/web-app-stalwart/files/test/test.sh new file mode 100755 index 0000000000..3b5d6f53de --- /dev/null +++ b/roles/web-app-stalwart/files/test/test.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Stalwart e2e coordinator: test-e2e-cli runs this with test.env exported, and each +# scenario below inherits that env, self-skipping when its scenario is not deployed. +set -euo pipefail +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +bash "${SCRIPT_DIR}/verify_migration.sh" diff --git a/roles/web-app-stalwart/files/test/verify_migration.sh b/roles/web-app-stalwart/files/test/verify_migration.sh new file mode 100644 index 0000000000..3699db0093 --- /dev/null +++ b/roles/web-app-stalwart/files/test/verify_migration.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Mailu → Stalwart migration, end to end against the deployed stack, entirely inside the +# DiD host: mail is delivered into the legacy Mailu, the import script moves it into +# Stalwart, and the same messages are then read back out of Stalwart over IMAP. +# No redeploy is involved — the cutover is a docker-level data move. +# nocheck: raw-docker — storage assertions read the maildir volume via the container wrapper +set -euo pipefail + +: "${STALWART_MIGRATION_E2E:?missing STALWART_MIGRATION_E2E}" +if [[ "${STALWART_MIGRATION_E2E}" != "true" ]]; then + echo "SKIP: web-app-mailu is not co-deployed; nothing to migrate." + exit 0 +fi + +: "${PYTHON_BIN:?missing PYTHON_BIN}" +: "${MIGRATION_SCRIPT:?missing MIGRATION_SCRIPT}" +: "${ADMIN_EMAIL:?missing ADMIN_EMAIL}" +: "${ADMIN_IMAP_PASSWORD:?missing ADMIN_IMAP_PASSWORD}" +: "${BIBER_EMAIL:?missing BIBER_EMAIL}" +: "${BIBER_IMAP_PASSWORD:?missing BIBER_IMAP_PASSWORD}" +: "${MAILU_MAILDIR_VOLUME:?missing MAILU_MAILDIR_VOLUME}" +: "${MAILU_SMTP_CONTAINER:?missing MAILU_SMTP_CONTAINER}" +: "${MAIL_DOMAIN:?missing MAIL_DOMAIN}" + +RUN_ID="$(date +%s)" +SUBJ_A="infinito-mig-A-${RUN_ID}" +SUBJ_B="infinito-mig-B-${RUN_ID}" +SUBJ_C="infinito-mig-C-${RUN_ID}" +SUBJ_D="infinito-mig-D-${RUN_ID}" + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +WORKDIR="$(mktemp -d)" +cleanup() { rm -rf "${WORKDIR}"; } +trap cleanup EXIT + +compose_mail() { + local from="$1" rcpt="$2" subject="$3" in_reply_to="${4:-}" + { + printf 'From: %s\r\nTo: %s\r\nSubject: %s\r\n' "${from}" "${rcpt}" "${subject}" + printf 'Message-ID: <%s@%s>\r\nDate: %s\r\n' "${subject}" "${MAIL_DOMAIN}" "$(date -R)" + if [[ -n "${in_reply_to}" ]]; then + printf 'In-Reply-To: <%s@%s>\r\n' "${in_reply_to}" "${MAIL_DOMAIN}" + fi + printf '\r\n%s\r\n' "migration e2e body ${subject}" + } >"${WORKDIR}/mail.eml" +} + +send_smtp() { + local host="$1" from="$2" rcpt="$3" subject="$4" attempt + for attempt in 1 2 3 4 5 6; do + if "${PYTHON_BIN}" "${SCRIPT_DIR}/probe_mail.py" send \ + "${host}" "${from}" "${rcpt}" "${WORKDIR}/mail.eml"; then + echo "sent: ${subject} (${from} -> ${rcpt} via ${host})" + return 0 + fi + echo "retry ${attempt}: submission of ${subject} to ${host} failed; waiting 10s" + sleep 10 + done + echo "FAIL: could not submit ${subject} to ${host}" >&2 + return 1 +} + +wait_stored_in_maildir() { + local subject="$1" mountpoint="$2" attempt + for attempt in $(seq 1 30); do + if grep -rqF "Subject: ${subject}" "${mountpoint}" 2>/dev/null; then + echo "stored: ${subject} under ${mountpoint}" + return 0 + fi + sleep 5 + done + echo "FAIL: ${subject} never appeared under ${mountpoint}" >&2 + return 1 +} + +wait_in_imap() { + local user="$1" password="$2" subject="$3" attempt + for attempt in $(seq 1 30); do + if "${PYTHON_BIN}" "${SCRIPT_DIR}/probe_mail.py" find \ + 127.0.0.1 "${user}" "${password}" "${subject}"; then + echo "found: ${subject} in ${user}'s mailbox" + return 0 + fi + sleep 5 + done + echo "FAIL: ${subject} not found over IMAP for ${user}" >&2 + return 1 +} + +# The migration is a pure docker-level move: read the legacy maildir volume, APPEND over +# IMAP. No redeploy is involved, so the harness invokes the very same script (and argv +# shape) that tasks/12_import_mailu.yml runs on a real cutover. +run_migration() { + local maildir="$1" + "${PYTHON_BIN}" -c \ + 'import json,sys; json.dump({sys.argv[1]: sys.argv[2], sys.argv[3]: sys.argv[4]}, open(sys.argv[5], "w"))' \ + "${ADMIN_EMAIL}" "${ADMIN_IMAP_PASSWORD}" \ + "${BIBER_EMAIL}" "${BIBER_IMAP_PASSWORD}" \ + "${WORKDIR}/accounts.json" + chmod 600 "${WORKDIR}/accounts.json" + + "${PYTHON_BIN}" "${MIGRATION_SCRIPT}" \ + --maildir-root "${maildir}" \ + --imap-host 127.0.0.1 --imap-port 993 --imap-insecure \ + --accounts-file "${WORKDIR}/accounts.json" + + rm -f "${WORKDIR}/accounts.json" +} + +echo "=== [1/4] Deliver mail into the legacy Mailu (A: biber->admin, B: admin's reply) ===" +MAILU_SMTP_IP="$(container inspect --type container -f '{{ range .NetworkSettings.Networks }}{{ .IPAddress }} {{ end }}' "${MAILU_SMTP_CONTAINER}" | awk '{print $1}')" +case "${MAILU_SMTP_IP}" in +[0-9]*.[0-9]*.[0-9]*.[0-9]*) ;; +*) + echo "FAIL: ${MAILU_SMTP_CONTAINER} has no IPv4 address (got '${MAILU_SMTP_IP}'); the legacy Mailu stack is not running" >&2 + exit 1 + ;; +esac +echo "mailu smtp at ${MAILU_SMTP_IP}" +compose_mail "${BIBER_EMAIL}" "${ADMIN_EMAIL}" "${SUBJ_A}" +send_smtp "${MAILU_SMTP_IP}" "${BIBER_EMAIL}" "${ADMIN_EMAIL}" "${SUBJ_A}" +compose_mail "${ADMIN_EMAIL}" "${BIBER_EMAIL}" "${SUBJ_B}" "${SUBJ_A}" +send_smtp "${MAILU_SMTP_IP}" "${ADMIN_EMAIL}" "${BIBER_EMAIL}" "${SUBJ_B}" + +echo "=== [2/4] Confirm the messages are stored in Mailu's maildir ===" +MAILDIR_MOUNT="$(container volume inspect --format '{{ .Mountpoint }}' "${MAILU_MAILDIR_VOLUME}")" +wait_stored_in_maildir "${SUBJ_A}" "${MAILDIR_MOUNT}/${ADMIN_EMAIL}" +wait_stored_in_maildir "${SUBJ_B}" "${MAILDIR_MOUNT}/${BIBER_EMAIL}" + +echo "=== [3/4] Migrate: run the import script against the deployed stack ===" +run_migration "${MAILDIR_MOUNT}" + +echo "=== [4/4] Continuity in Stalwart, then live flow (C: biber->admin, D: admin's reply) ===" +wait_in_imap "${ADMIN_EMAIL}" "${ADMIN_IMAP_PASSWORD}" "${SUBJ_A}" +wait_in_imap "${BIBER_EMAIL}" "${BIBER_IMAP_PASSWORD}" "${SUBJ_B}" +compose_mail "${BIBER_EMAIL}" "${ADMIN_EMAIL}" "${SUBJ_C}" +send_smtp "127.0.0.1" "${BIBER_EMAIL}" "${ADMIN_EMAIL}" "${SUBJ_C}" +compose_mail "${ADMIN_EMAIL}" "${BIBER_EMAIL}" "${SUBJ_D}" "${SUBJ_C}" +send_smtp "127.0.0.1" "${ADMIN_EMAIL}" "${BIBER_EMAIL}" "${SUBJ_D}" +wait_in_imap "${ADMIN_EMAIL}" "${ADMIN_IMAP_PASSWORD}" "${SUBJ_C}" +wait_in_imap "${BIBER_EMAIL}" "${BIBER_IMAP_PASSWORD}" "${SUBJ_D}" + +echo "MIGRATION STATE MACHINE COMPLETE" diff --git a/roles/web-app-stalwart/lookup_plugins/stalwart_users.py b/roles/web-app-stalwart/lookup_plugins/stalwart_users.py new file mode 100644 index 0000000000..631d101b0e --- /dev/null +++ b/roles/web-app-stalwart/lookup_plugins/stalwart_users.py @@ -0,0 +1,89 @@ +"""Lookup ``stalwart_users``: the subset of the user catalogue Stalwart provisions. + +``lookup('users')`` yields every user any role declares -- 295 of them in a full +CI round -- while Stalwart acts on seven. The rest are the ``accounts: []`` name +reservations ``utils.cache.users`` mints for every role-name suffix; creating a +JMAP account for one contradicts the empty declaration and hands an +authenticable SMTP/IMAP principal to a name that exists only to block squatting. + +A user is Stalwart's when it carries a mailbox account, or when it is a bot, +whose submission credential ``sys-token-store`` persists. Both halves are +needed: the token task is gated on the role, not on the account, so filtering by +mailbox alone would silently stop issuing a credential to a bot that has no +mailbox of its own. + +Mirrors ``roles/web-app-mailu/lookup_plugins/mailu_users.py`` so the two mail +providers select the same set from the same contract. + +Returns the same dict shape ``lookup('users')`` yields, so call sites keep +``| dict2items`` and ``user_item.value.*`` unchanged. + +Usage, through the constant in ``vars/main.yml``: + + STALWART_USERS: "{{ lookup('stalwart_users') }}" +""" + +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleError +from ansible.plugins.loader import lookup_loader +from ansible.plugins.lookup import LookupBase + +MAILBOX_ACCOUNT = "mailbox" +BOT_ROLE = "bot" + + +def _string_list(user: dict[str, Any], key: str) -> list[str]: + """Read one list-valued user attribute, defaulting to empty. + + Args: + user: one entry of the merged user catalogue. + key: attribute name, ``accounts`` or ``roles``. + + Raises: + AnsibleError: the attribute is present but is not a list. + """ + value = user.get(key, []) + if value is None: + return [] + if not isinstance(value, (list, tuple)): + raise AnsibleError( + f"stalwart_users: user '{user.get('username', '?')}' declares " + f"{key}={value!r}; a list is required." + ) + return [str(entry) for entry in value] + + +def provisioned_by_stalwart(user: dict[str, Any]) -> bool: + """Return whether Stalwart acts on this user at all.""" + return MAILBOX_ACCOUNT in _string_list( + user, "accounts" + ) or BOT_ROLE in _string_list(user, "roles") + + +class LookupModule(LookupBase): + def run( + self, + terms: list[Any] | None, + variables: dict[str, Any] | None = None, + **kwargs: Any, + ) -> list[dict[str, Any]]: + if terms: + raise AnsibleError("stalwart_users lookup takes no positional terms") + + variables = variables or getattr(self._templar, "available_variables", {}) or {} + users = lookup_loader.get( + "users", + loader=getattr(self, "_loader", None), + templar=getattr(self, "_templar", None), + ).run([], variables=variables, **kwargs)[0] + + return [ + { + key: user + for key, user in users.items() + if isinstance(user, dict) and provisioned_by_stalwart(user) + } + ] diff --git a/roles/web-app-stalwart/meta/csp.yml b/roles/web-app-stalwart/meta/csp.yml new file mode 100644 index 0000000000..fdf8f27457 --- /dev/null +++ b/roles/web-app-stalwart/meta/csp.yml @@ -0,0 +1,12 @@ +--- +flags: + style-src-attr: + unsafe-inline: true # nocheck: csp-unsafe Reason: the WebAdmin SPA lays out its dialogs, tables and progress bars by writing `style="..."` attributes from JS; without it the settings and account views render unstyled and overlap. + script-src-elem: + unsafe-inline: true # nocheck: csp-unsafe Reason: Stalwart's WebAdmin index.html boots the SPA from an inline