diff --git a/.ansible-lint b/.ansible-lint index 2cf3aab2b9..567e47480a 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -26,6 +26,7 @@ kinds: - tasks: "roles/*/tasks/*.yml" - tasks: "tasks/**/*.yml" - vars: "roles/*/vars/*.yml" + - vars: "roles/*/meta/addons/*.yml" - vars: "group_vars/**/*.yml" - vars: "host_vars/**/*.yml" - meta: "roles/*/meta/main.yml" diff --git a/.github/workflows/ci-orchestrator.yml b/.github/workflows/ci-orchestrator.yml index 4a8e990b61..720572ce1b 100644 --- a/.github/workflows/ci-orchestrator.yml +++ b/.github/workflows/ci-orchestrator.yml @@ -27,6 +27,11 @@ on: required: false type: string default: "false" + trusted_pr: + description: "True when the PR carries the maintainer trusted-pr label; skips the fork-prereq wait." + required: false + type: boolean + default: false concurrency: group: infinito-test-${{ github.ref }} @@ -42,23 +47,25 @@ jobs: wait-fork-prereq-run: name: โณ Wait for fork run runs-on: ubuntu-latest + env: + WAIT_REQUIRED: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && !inputs.trusted_pr }} steps: - - name: Mark fork prerequisites as ready outside fork PRs - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + - name: Mark fork prerequisites as ready outside fork PRs or for trusted PRs + if: ${{ env.WAIT_REQUIRED != 'true' }} shell: bash run: | set -euo pipefail - echo "Fork prerequisites are implicitly ready outside fork pull_request events." + echo "Fork prerequisites are implicitly ready (non-fork PR or trusted-pr label)." - name: Checkout trusted workflow repo (helpers) - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + if: ${{ env.WAIT_REQUIRED == 'true' }} uses: actions/checkout@v7 with: ref: ${{ github.event.pull_request.base.sha }} persist-credentials: false - name: Login to GHCR for CI image checks - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + if: ${{ env.WAIT_REQUIRED == 'true' }} uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io @@ -66,7 +73,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Wait for privileged fork CI images - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + if: ${{ env.WAIT_REQUIRED == 'true' }} shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -115,6 +122,8 @@ jobs: uses: ./.github/workflows/lint-markdown.yml lint-makefile: uses: ./.github/workflows/lint-makefile.yml + lint-packages: + uses: ./.github/workflows/lint-packages.yml lint-javascript: uses: ./.github/workflows/lint-javascript.yml lint-playwright: @@ -128,6 +137,7 @@ jobs: - lint-shell - lint-markdown - lint-makefile + - lint-packages - lint-javascript - lint-playwright runs-on: ubuntu-latest diff --git a/.github/workflows/cleanup-stale.yml b/.github/workflows/cleanup-stale.yml new file mode 100644 index 0000000000..7de697df62 --- /dev/null +++ b/.github/workflows/cleanup-stale.yml @@ -0,0 +1,89 @@ +name: "๐Ÿงน Cleanup: Stale Repository Data" + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + +permissions: + contents: write + issues: write + pull-requests: write + packages: write + +jobs: + stale: + name: ๐Ÿท๏ธ Close inactive issues and pull requests + runs-on: ubuntu-latest + + steps: + - name: ๐Ÿท๏ธ Mark stale issues and pull requests + uses: actions/stale@v9 + with: + # Issues + days-before-issue-stale: 90 + days-before-issue-close: 7 + stale-issue-label: "stale" + stale-issue-message: > + This issue has been marked as stale because it has had no activity + for the last 90 days. It will be automatically closed in 7 days + unless there is further activity. + close-issue-message: > + This issue has been automatically closed due to inactivity. + + # Pull Requests + days-before-pr-stale: 180 + days-before-pr-close: 30 + stale-pr-label: "stale" + stale-pr-message: > + This pull request has been marked as stale because it has had no + activity for the last 180 days. It will be automatically closed in + 30 days unless there is further activity. + close-pr-message: > + This pull request has been automatically closed due to inactivity. + + remove-stale-when-updated: true + + delete-stale-branches: + name: ๐ŸŒฟ Delete inactive branches + runs-on: ubuntu-latest + + steps: + - name: ๐ŸŒฟ Delete branches inactive for 360 days + uses: crs-k/stale-branches@v8 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + days-before-stale: 360 + days-before-delete: 0 + + protected-branches: | + main + master + develop + dev + release/** + hotfix/** + + compare-branches: info + dry-run: false + + cleanup-ghcr-ci-images: + name: ๐Ÿ“ฆ Delete old GHCR CI images + runs-on: ubuntu-latest + + steps: + - name: ๐Ÿ“ฅ Checkout + uses: actions/checkout@v7 + + - name: ๐Ÿ“ฆ Install project environment + run: make install + + - name: ๐Ÿ“ฆ Delete ci-* images older than 60 days + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ORG: infinito-nexus + PACKAGE_PREFIX: core + MAX_AGE_DAYS: 60 + TAG_PREFIX: ci- + run: bash scripts/github/cleanup/ghcr_ci_images.sh diff --git a/.github/workflows/entry-pull-request-change.yml b/.github/workflows/entry-pull-request-change.yml index bc8c0ebe5b..b0a267eaf3 100644 --- a/.github/workflows/entry-pull-request-change.yml +++ b/.github/workflows/entry-pull-request-change.yml @@ -9,12 +9,14 @@ on: - synchronize - reopened - ready_for_review + - labeled pull_request_target: types: - opened - synchronize - reopened - ready_for_review + - labeled concurrency: # Keep only the newest run per PR and event type. @@ -128,6 +130,7 @@ jobs: distros: "arch debian ubuntu fedora centos" whitelist: ${{ needs.detect-affected-roles.outputs.whitelist }} roles_only: ${{ needs.detect-affected-roles.outputs.roles_only }} + trusted_pr: ${{ contains(github.event.pull_request.labels.*.name, 'trusted-pr') }} prepare-fork-merge-ref: needs: @@ -138,6 +141,7 @@ jobs: outputs: checkout_ref: ${{ steps.out.outputs.checkout_ref }} image_tag: ${{ steps.out.outputs.image_tag }} + trusted: ${{ contains(github.event.pull_request.labels.*.name, 'trusted-pr') }} steps: - name: Checkout uses: actions/checkout@v7 @@ -154,7 +158,8 @@ jobs: uses: ./.github/workflows/images-build-ci.yml with: distros: "arch debian ubuntu fedora centos" - checkout_ref: ${{ needs.prepare-fork-merge-ref.outputs.checkout_ref }} + checkout_ref: ${{ needs.prepare-fork-merge-ref.outputs.trusted == 'true' && github.event.pull_request.head.sha || needs.prepare-fork-merge-ref.outputs.checkout_ref }} + source_repository: ${{ needs.prepare-fork-merge-ref.outputs.trusted == 'true' && github.event.pull_request.head.repo.full_name || github.repository }} image_tag: ${{ needs.prepare-fork-merge-ref.outputs.image_tag }} concurrency_channel: pr-target-build-producer secrets: inherit diff --git a/.github/workflows/entry-push-latest.yml b/.github/workflows/entry-push-latest.yml index e9c11bdd25..ab93c1601d 100644 --- a/.github/workflows/entry-push-latest.yml +++ b/.github/workflows/entry-push-latest.yml @@ -22,11 +22,29 @@ permissions: security-events: write jobs: + sync-main: + name: ๐Ÿ”„ Sync main + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Sync main from source repository + shell: bash + env: + CI_SYNC_MAIN_SOURCE_REPOSITORY_DEFAULT: infinito-nexus/core + CI_SYNC_MAIN_SOURCE_REPOSITORY: ${{ vars.CI_SYNC_MAIN_SOURCE_REPOSITORY }} + CI_SYNC_MAIN_SOURCE_REPOSITORY_IS_SET: ${{ contains(toJSON(vars), '"CI_SYNC_MAIN_SOURCE_REPOSITORY"') }} + run: ./scripts/github/sync/main_from_repository.sh + # Skip CI when the push is to main and CI_RUN_ON_MAIN is not 'true', or # when the push is a branch creation whose commit is already in origin/main. # All other pushes run CI. run-policy: name: ๐Ÿงญ Decide CI policy + needs: sync-main runs-on: ubuntu-latest outputs: should_run: ${{ steps.decide.outputs.should_run }} diff --git a/.github/workflows/images-build-ci.yml b/.github/workflows/images-build-ci.yml index 477d9d3f71..c90d0d7e24 100644 --- a/.github/workflows/images-build-ci.yml +++ b/.github/workflows/images-build-ci.yml @@ -12,6 +12,11 @@ on: required: false type: string default: "" + source_repository: + description: "Repository to checkout as the build source (default: github.repository). Set to a fork only for maintainer-trusted PRs." + required: false + type: string + default: "" image_tag: description: "Optional image tag override (default: ci-${github.sha})" required: false @@ -68,7 +73,7 @@ jobs: - name: Checkout source to build uses: actions/checkout@v7 with: - repository: ${{ github.repository }} + repository: ${{ inputs.source_repository || github.repository }} ref: ${{ inputs.checkout_ref || github.ref }} path: _src persist-credentials: false diff --git a/.github/workflows/images-cleanup-ci.yml b/.github/workflows/images-cleanup-ci.yml index 1c4564e0e6..397ebadb07 100644 --- a/.github/workflows/images-cleanup-ci.yml +++ b/.github/workflows/images-cleanup-ci.yml @@ -34,10 +34,11 @@ jobs: shell: bash run: | set -euo pipefail + source scripts/meta/env/load.sh { echo "owner=$(scripts/meta/resolve/repository/owner.sh)" echo "repo_prefix=$(scripts/meta/resolve/repository/name.sh)" - echo "distros=$(scripts/meta/resolve/distros.sh)" + echo "distros=${INFINITO_DISTROS}" } >> "$GITHUB_OUTPUT" - name: ๐Ÿ“ฆ Install jq diff --git a/.github/workflows/lint-packages.yml b/.github/workflows/lint-packages.yml new file mode 100644 index 0000000000..287ec8e330 --- /dev/null +++ b/.github/workflows/lint-packages.yml @@ -0,0 +1,25 @@ +name: "๐Ÿ” Lint: Packages" + +on: + workflow_call: + +permissions: + contents: read + +jobs: + lint-packages: + name: ๐Ÿงน Lint Packages + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install packaging validators + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends dpkg-dev rpm + + - name: Validate distro packaging metadata + run: bash scripts/lint/packages.sh diff --git a/.github/workflows/release-highest.yml b/.github/workflows/release-highest.yml index 642271b046..c22176329b 100644 --- a/.github/workflows/release-highest.yml +++ b/.github/workflows/release-highest.yml @@ -34,13 +34,13 @@ jobs: shell: bash run: | set -euo pipefail + source scripts/meta/env/load.sh owner="$(scripts/meta/resolve/repository/owner.sh)" repo_prefix="$(scripts/meta/resolve/repository/name.sh)" - distros="$(scripts/meta/resolve/distros.sh)" { echo "owner=${owner}" echo "repo_prefix=${repo_prefix}" - echo "distros=${distros}" + echo "distros=${INFINITO_DISTROS}" } >> "$GITHUB_OUTPUT" - name: Login to GHCR (for manifest checks) diff --git a/.github/workflows/test-deploy-local.yml b/.github/workflows/test-deploy-local.yml index a335e0c766..96c83d826a 100644 --- a/.github/workflows/test-deploy-local.yml +++ b/.github/workflows/test-deploy-local.yml @@ -82,7 +82,7 @@ jobs: run: ./scripts/github/resolve/output_apps.sh deploy: - name: ๐Ÿš€ Deploy ${{ needs.discover.outputs.test_deploy_type }} app (${{ matrix.apps }}) + name: ๐Ÿš€ Deploy ${{ needs.discover.outputs.test_deploy_type }} app (${{ matrix.apps }}${{ matrix.variant && format(' [{0}]', matrix.variant) || '' }}) if: needs.discover.outputs.apps != '' && needs.discover.outputs.apps != '[]' runs-on: ubuntu-latest timeout-minutes: 360 @@ -92,7 +92,7 @@ jobs: fail-fast: false max-parallel: 1 # immer linear matrix: - apps: ${{ fromJson(needs.discover.outputs.apps) }} + include: ${{ fromJson(needs.discover.outputs.apps) }} steps: # Reclaim runner space before Docker pulls start. Deploy jobs otherwise @@ -128,6 +128,7 @@ jobs: INFINITO_DEPLOY_TYPE: ${{ needs.discover.outputs.test_deploy_type }} INFINITO_DISTROS: ${{ needs.discover.outputs.distros }} apps: ${{ matrix.apps }} + variant: ${{ matrix.variant }} ANSIBLE_LOG_PATH: /tmp/ansible-${{ matrix.apps }}.log run: ./scripts/tests/deploy/ci/all.sh diff --git a/.github/workflows/test-deploy-server.yml b/.github/workflows/test-deploy-server.yml index ba3795dfaa..3e992262e3 100644 --- a/.github/workflows/test-deploy-server.yml +++ b/.github/workflows/test-deploy-server.yml @@ -75,7 +75,7 @@ jobs: run: ./scripts/github/resolve/output_apps.sh server: - name: ๐ŸŒ Deploy ${{ matrix.apps }} + name: ๐ŸŒ Deploy ${{ matrix.apps }}${{ matrix.variant && format(' [{0}]', matrix.variant) || '' }} if: needs.discover.outputs.apps != '' && needs.discover.outputs.apps != '[]' runs-on: ubuntu-latest timeout-minutes: 360 @@ -84,7 +84,7 @@ jobs: strategy: fail-fast: false matrix: - apps: ${{ fromJson(needs.discover.outputs.apps) }} + include: ${{ fromJson(needs.discover.outputs.apps) }} steps: # Reclaim runner space before Docker pulls start. Server deploy tests pull @@ -125,6 +125,7 @@ jobs: shell: bash env: apps: ${{ matrix.apps }} + variant: ${{ matrix.variant }} ANSIBLE_LOG_PATH: /tmp/ansible-${{ matrix.apps }}.log run: ./scripts/tests/deploy/ci/all.sh @@ -152,6 +153,6 @@ jobs: if: always() uses: actions/upload-artifact@v7 with: - name: playwright-server-${{ matrix.apps }} + name: playwright-server-${{ matrix.apps }}${{ matrix.variant_slug && format('-{0}', matrix.variant_slug) || '' }} path: /tmp/playwright-artifacts/ if-no-files-found: ignore diff --git a/.github/workflows/test-deploy-universal.yml b/.github/workflows/test-deploy-universal.yml index 5de2c31de2..3488ce723f 100644 --- a/.github/workflows/test-deploy-universal.yml +++ b/.github/workflows/test-deploy-universal.yml @@ -85,7 +85,7 @@ jobs: run: bash scripts/github/split_runner_apps.sh universal: - name: โœจ Deploy ${{ matrix.apps }} + name: โœจ Deploy ${{ matrix.apps }}${{ matrix.variant && format(' [{0}]', matrix.variant) || '' }} if: needs.discover.outputs.apps_github != '' && needs.discover.outputs.apps_github != '[]' runs-on: ubuntu-latest timeout-minutes: 360 @@ -94,7 +94,7 @@ jobs: strategy: fail-fast: false matrix: - apps: ${{ fromJson(needs.discover.outputs.apps_github) }} + include: ${{ fromJson(needs.discover.outputs.apps_github) }} steps: # Reclaim runner space before Docker pulls start. Deploy jobs otherwise @@ -134,6 +134,7 @@ jobs: shell: bash env: apps: ${{ matrix.apps }} + variant: ${{ matrix.variant }} ANSIBLE_LOG_PATH: /tmp/ansible-${{ matrix.apps }}.log GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: ./scripts/tests/deploy/ci/all.sh @@ -149,7 +150,7 @@ jobs: run: ./scripts/github/runner/dump_state.sh "post-deploy failure (${{ matrix.apps }})" universal-self-hosted: - name: โœจ Deploy ${{ matrix.apps }} (self-hosted) + name: โœจ Deploy ${{ matrix.apps }}${{ matrix.variant && format(' [{0}]', matrix.variant) || '' }} (self-hosted) if: needs.discover.outputs.apps_self_hosted != '' && needs.discover.outputs.apps_self_hosted != '[]' runs-on: [self-hosted, linux] timeout-minutes: 360 @@ -158,7 +159,7 @@ jobs: strategy: fail-fast: false matrix: - apps: ${{ fromJson(needs.discover.outputs.apps_self_hosted) }} + include: ${{ fromJson(needs.discover.outputs.apps_self_hosted) }} steps: - name: Fix workspace ownership @@ -177,6 +178,7 @@ jobs: shell: bash env: apps: ${{ matrix.apps }} + variant: ${{ matrix.variant }} INFINITO_TIMEOUT_MULTIPLIER: ${{ vars.INFINITO_TIMEOUT_MULTIPLIER || '1' }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: ./scripts/tests/deploy/ci/all.sh diff --git a/.github/workflows/test-deploy-workstation.yml b/.github/workflows/test-deploy-workstation.yml index f15afb9899..3f1eae938d 100644 --- a/.github/workflows/test-deploy-workstation.yml +++ b/.github/workflows/test-deploy-workstation.yml @@ -84,7 +84,7 @@ jobs: run: bash scripts/github/split_runner_apps.sh workstation: - name: ๐Ÿ–ฅ๏ธ Deploy ${{ matrix.apps }} + name: ๐Ÿ–ฅ๏ธ Deploy ${{ matrix.apps }}${{ matrix.variant && format(' [{0}]', matrix.variant) || '' }} if: needs.discover.outputs.apps_github != '' && needs.discover.outputs.apps_github != '[]' runs-on: ubuntu-latest timeout-minutes: 360 @@ -93,7 +93,7 @@ jobs: strategy: fail-fast: false matrix: - apps: ${{ fromJson(needs.discover.outputs.apps_github) }} + include: ${{ fromJson(needs.discover.outputs.apps_github) }} steps: # Reclaim runner space before Docker pulls start. Deploy jobs otherwise @@ -133,6 +133,7 @@ jobs: shell: bash env: apps: ${{ matrix.apps }} + variant: ${{ matrix.variant }} ANSIBLE_LOG_PATH: /tmp/ansible-${{ matrix.apps }}.log run: ./scripts/tests/deploy/ci/all.sh @@ -147,7 +148,7 @@ jobs: run: ./scripts/github/runner/dump_state.sh "post-deploy failure (${{ matrix.apps }})" workstation-self-hosted: - name: ๐Ÿ–ฅ๏ธ Deploy ${{ matrix.apps }} (self-hosted) + name: ๐Ÿ–ฅ๏ธ Deploy ${{ matrix.apps }}${{ matrix.variant && format(' [{0}]', matrix.variant) || '' }} (self-hosted) if: needs.discover.outputs.apps_self_hosted != '' && needs.discover.outputs.apps_self_hosted != '[]' runs-on: [self-hosted, linux] timeout-minutes: 360 @@ -156,7 +157,7 @@ jobs: strategy: fail-fast: false matrix: - apps: ${{ fromJson(needs.discover.outputs.apps_self_hosted) }} + include: ${{ fromJson(needs.discover.outputs.apps_self_hosted) }} steps: - name: Fix workspace ownership @@ -175,6 +176,7 @@ jobs: shell: bash env: apps: ${{ matrix.apps }} + variant: ${{ matrix.variant }} INFINITO_TIMEOUT_MULTIPLIER: ${{ vars.INFINITO_TIMEOUT_MULTIPLIER || '1' }} run: ./scripts/tests/deploy/ci/all.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dd9159dec..ca89ad64e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,50 @@ # Changelog +## [11.1.0] - 2026-06-27 + +* Trusted fork PRs: a maintainer-applied *trusted-pr* label runs a fork PR as deliberately released code โ€” the orchestrator skips the fork-prerequisite wait and the privileged *pull_request_target* build switches its source from the base merge ref to the raw PR head, while the built image keeps the merge-SHA tag so every downstream consumer is unchanged. The label is the only switch (applying a label is collaborator-only on GitHub, so fork authors cannot grant it), build-orchestration scripts always come from the base repository, and both unlabeled fork PRs and same-repository PRs are untouched. See [the fork-PR pipeline](docs/contributing/artefact/git/pipeline.md) and [the contributor fork workflow](docs/contributing/artefact/git/pull-request.md). + +* Fork main-branch sync: the push entry workflow now synchronizes the current repository's *main* from a configured source repository (default *infinito-nexus/core*) before CI scope and deploy discovery run, so a fork branch push sees only its own changes against an up-to-date base. The source is overridable or disablable through the *CI_SYNC_MAIN_SOURCE_REPOSITORY* repository variable, and the sync force-overwrites *main*, so a synced repository must not be used as a working branch. See [the Actions configuration reference](docs/contributing/tools/github/actions/configuration.md). + +* Package-changelog generator hardening: the Debian changelog and the Fedora spec *%changelog* are mirrored from CHANGELOG.md without tripping their native parsers โ€” every change line is indented for Debian, every body line gains a dash continuation for RPM, and the archived-releases notice is folded into the oldest kept entry instead of trailing after the final entry. Previously the un-indented bodies made *dpkg-buildpackage* abort the Debian package build with an empty maintainer. The *Changelog* title is also preserved when the active window is trimmed. See [the release procedure](docs/contributing/actions/release.md). + +* New packaging lint: a *lint-packages* check, wired into *make lint* and its own CI workflow, validates the generated *debian/changelog* with *dpkg-parsechangelog*, the Fedora spec with *rpmspec* and the Arch *PKGBUILD* with a *bash -n* syntax pass, failing on any parser warning rather than only on a non-zero exit. A complementary *test-lint* check asserts that both package changelogs list every released version and that their inline window matches CHANGELOG.md. See [the workflow catalog](docs/contributing/tools/github/actions/workflows.md). + +* Repository cleanup automation: a scheduled cleanup workflow plus a GHCR helper prune stale CI images so the container registry does not accumulate per-commit build artefacts. + +* Routine maintenance: YAML normalization of the *web-app-pihole* role variables (document start, single-space keys). + +**Contributors** + +* [Kevin Veen-Birkenbach](https://veen.world): trusted-fork-PR flow, fork main sync, package-changelog generator and packaging lint, cleanup automation + +## [11.0.0] - 2026-06-27 + +* Unified addon syntax (requirement 026): every role-level plugin, browser/GNOME extension, Matrix bridge and cross-role integration was migrated to a single declarative *meta/addons/* contract, replacing the per-role ad-hoc plugin definitions. 86 addon files now describe addons for *desk-chromium*, *desk-firefox*, *desk-gnome*, *web-app-discourse*, *web-app-friendica*, *web-app-joomla*, *web-app-mediawiki*, *web-app-pretix*, *web-app-wordpress*, *web-app-xwiki*, *web-app-nextcloud*, *web-app-odoo* and *web-app-matrix*. Each service-coupled addon gets a *tasks/addons* integration hook and its own Playwright spec (87 per-addon specs added), addon flags are gated on actual partner deployment through a new *addon_env_flags* lookup, and a lint requires a hook plus a spec for every coupled addon. The cross-role integration matrix is regenerated from a dedicated data module (*integration_matrix_data.py*). +* New centralized API single-point-of-truth: an *api* lookup plugin plus a global API dict so API tokens and endpoints (Cloudflare, the Nextcloud proprietary OAuth integrations) resolve from one place instead of scattered role lookups. +* Real Nextcloud integration coupling: end-to-end tests now assert genuine two-way OAuth/API coupling for *gitlab*, *mattermost*, *openproject*, *zammad*, *peertube*, *discourse*, *github*, *google*, *jira*, *mastodon*, *matrix*, *moodle* and *suitecrm*, rather than the mere presence of a connect button. All integration OAuth credentials are stored Nextcloud-encrypted via ICrypto (sensitive app values), OpenProject and Zammad secrets survive a re-deploy, and OpenWebUI/Ollama is wired in as a real *integration_openai* chat backend. Bridges with no Nextcloud-33 plugin (Moodle, SuiteCRM) are disabled. +* Nextcloud deploy-variant rebalancing: Discourse and OnlyOffice are each isolated into their own loosely-coupled partner variant, the matrix is repacked to five variants overriding only the dynamic flags, LDAP is enabled wherever SSO is on, and the database is shared outside variant 1. The GitLab 19 OAuth app now carries an organization, and *fileslibreofficeedit* is gated on OnlyOffice absence. +* Storage- and runtime-aware CI variant matrix: variant bundles are split per runner by a storage-aware job splitter driven by an explicit variant CSV, behind a unified variant selector (*variant_select.py*). The default per-bundle storage cap is centralized at 350GB, per-variant memory budgets are guarded against the host budget, shared database providers are enforced by a shared-false-once lint, and the full variant matrix always runs (the earlier runtime time-cut was removed). +* Container resource governance: self-run and shared-provider services declare explicit compose mem/pids limits, container limits were right-sized across roles, and Node-heap services respect a floor via a new *node_max_old_space_size* lookup (Nextcloud whiteboard raised to 1g). A reworked *ressources* CLI emits per-variant and all-role resource summaries, the footprint tool gains a *min_storage* column, and shared services carry a *bond* integration-importance factor. +* Matrix bridges modernized: Signal, Telegram, WhatsApp and Slack bridges migrate to bridgev2 config, the dead mautrix-facebook/instagram bridges are replaced by mautrix-meta, bridge specs are gated by the deployed addon, Synapse loads the bridge registrations from a mounted directory, and the role now tracks *matrix-docker-ansible-deploy* from the spantaleev upstream. +* CA-trust and OIDC-over-self-signed-TLS hardening: the internal CA is installed into the OS bundle for PHP libcurl OIDC (*web-app-pixelfed*, *web-app-joomla*, plus a WordPress must-use plugin), the CA-trust wrapper is kept on probe timeout instead of degrading to env-only, and the CA injection is bounded so it cannot hang the handler. HSTS is now emitted on every response including non-2xx, *ansible-vault encrypt_string* reads plaintext from stdin so leading-dash values encrypt, and Mastodon allows inline *script-src-elem* for its server-rendered SPA. +* Image and dependency version jumps (net since 10.1.1): + * *web-app-erpnext*: v16.23.1 to v16.25.0 + * *web-app-gitlab*: 19.1.0-ee.0 to 19.1.1-ee.0 + * *web-app-opencloud*: 4.0.7 to 7.2.0 + * *web-app-seaweedfs*: 4.35 to 4.36 + * *web-app-matomo*: unpinned *latest* to pinned 5.3.2 (matches the bootstrap installer) + * *web-app-nextcloud* proxy (nginx): unpinned *alpine* to pinned 1.31.2-alpine + * *web-app-nextcloud* whiteboard: unpinned *latest* to pinned v1.5.9 + * *svc-ai-ollama*: switched to the official *ollama* image so inference works + * *web-app-pretix* and *web-app-xwiki*: their inline plugin/extension version pins (2.3.1; 2.19.6, 9.15.7, 1.0) were removed as those addons moved into *meta/addons* + * Dev/CI dependencies: *globals* 15.15.0 to 17.7.0, *@playwright/test* 1.61.0 to 1.61.1, *actions/cache* 5 to 6 + * Intermediate bot bumps were corrected back down: Mattermost 11.8.2 to 11.8.1 and Nextcloud 34 to 33 (kept at 33 because of plugin incompatibilities, same pin as 10.0.0) + +**Contributors** + +* [Kevin Veen-Birkenbach](https://veen.world): unified addon syntax, API single-point-of-truth, Nextcloud integrations and variant rebalancing, CI variant bundling, resource governance, Matrix bridge migration and version maintenance + ## [10.1.1] - 2026-06-24 * Disabled the *email* service in the *web-app-pihole* role by pinning *services.email* to a static *enabled: false* / *shared: false* with the matching *dynamic-flag* and *email* nocheck suppressions, removing the now-static email overrides from *meta/variants.yml*, and adding *javascript* true/false variant pins to keep the dynamic-flag matrix-coverage guard green. diff --git a/Makefile b/Makefile index 0290045818..4e58005445 100644 --- a/Makefile +++ b/Makefile @@ -332,6 +332,7 @@ lint: install-lint lint-javascript \ lint-makefile \ lint-markdown \ + lint-packages \ lint-playwright \ lint-python \ lint-shellcheck @@ -363,6 +364,13 @@ lint-makefile: install-lint lint-markdown: install-lint @bash scripts/lint/wrapper.sh markdown +.PHONY: lint-packages +# Validate distro packaging metadata (debian changelog, fedora spec, arch PKGBUILD). +# Note: provisions the native parsers explicitly, then validates; absent tools are skipped. +lint-packages: install-lint + @bash scripts/install/wrapper.sh packages + @bash scripts/lint/wrapper.sh packages + .PHONY: lint-playwright # Verify every role's Playwright spec parses + resolves its helpers. # Note: stages the spec like test-e2e-playwright does and runs `npx playwright test --list`. diff --git a/cli/administration/deploy/development/deploy.py b/cli/administration/deploy/development/deploy.py index 76be4306dd..f851d11477 100644 --- a/cli/administration/deploy/development/deploy.py +++ b/cli/administration/deploy/development/deploy.py @@ -16,24 +16,13 @@ from .common import ( make_compose, resolve_container, ) -from .inventory import filter_plan_to_variant, plan_dev_inventory_matrix +from .inventory import plan_dev_inventory_matrix +from .variant_select import add_variant_args, apply_variant_filter if TYPE_CHECKING: from collections.abc import Mapping -def _env_variant() -> int | None: - raw = os.environ.get("variant", "").strip() - if not raw: - return None - try: - return int(raw) - except ValueError: - raise SystemExit( - f"variant environment variable must be an integer, got {raw!r}" - ) from None - - def _env_full_cycle() -> bool: return os.environ.get("full_cycle", "").strip().lower() == "true" @@ -188,18 +177,7 @@ def add_parser(sub: argparse._SubParsersAction) -> None: default=False, help="Enable/disable Ansible debug mode (default: disabled).", ) - p.add_argument( - "--variant", - type=int, - default=_env_variant(), - help=( - "Pin the matrix deploy to a single round (zero-based index), " - "skipping inter-round cleanup. Useful for redeploying one " - "specific variant without iterating the whole matrix. Defaults " - "to the variant environment variable when set, otherwise " - "full-matrix mode." - ), - ) + add_variant_args(p, action="deploy") p.add_argument( "--full-cycle", action=argparse.BooleanOptionalAction, @@ -273,7 +251,7 @@ def handler(args: argparse.Namespace) -> int: base_inventory_dir=str(args.inventory_dir), ) try: - plan = filter_plan_to_variant(plan, args.variant) + plan = apply_variant_filter(plan, args) except ValueError as exc: raise SystemExit(f"--variant: {exc}") from exc diff --git a/cli/administration/deploy/development/init.py b/cli/administration/deploy/development/init.py index 5f4df3f6da..516ae2d0f7 100644 --- a/cli/administration/deploy/development/init.py +++ b/cli/administration/deploy/development/init.py @@ -14,10 +14,10 @@ from .common import make_compose from .inventory import ( DevInventorySpec, build_dev_inventory, - filter_plan_to_variant, plan_dev_inventory_matrix, ) from .storage import detect_storage_constrained +from .variant_select import add_variant_args, apply_variant_filter if TYPE_CHECKING: import argparse @@ -67,32 +67,10 @@ def add_parser(sub: argparse._SubParsersAction) -> None: default=None, help="JSON object merged into inventory variables (overrides win).", ) - p.add_argument( - "--variant", - type=int, - default=_env_variant(), - help=( - "Pin the matrix init to a single round (zero-based index). " - "Useful when only one variant of a multi-variant app needs to " - "be (re-)materialised. Defaults to the variant " - "environment variable when set, otherwise full-matrix mode." - ), - ) + add_variant_args(p, action="init") p.set_defaults(_handler=handler) -def _env_variant() -> int | None: - raw = os.environ.get("variant", "").strip() - if not raw: - return None - try: - return int(raw) - except ValueError: - raise SystemExit( - f"variant environment variable must be an integer, got {raw!r}" - ) from None - - def handler(args: argparse.Namespace) -> int: compose = make_compose() @@ -154,7 +132,7 @@ def handler(args: argparse.Namespace) -> int: base_inventory_dir=str(args.inventory_dir), ) try: - plan = filter_plan_to_variant(plan, args.variant) + plan = apply_variant_filter(plan, args) except ValueError as exc: raise SystemExit(f"--variant: {exc}") from exc diff --git a/cli/administration/deploy/development/inventory/__init__.py b/cli/administration/deploy/development/inventory/__init__.py index 2b7c900de6..2e0181fd10 100644 --- a/cli/administration/deploy/development/inventory/__init__.py +++ b/cli/administration/deploy/development/inventory/__init__.py @@ -35,7 +35,7 @@ from __future__ import annotations from .builder import build_dev_inventory, build_dev_inventory_matrix from .legacy_resolver import _build_services_overrides_for_round, _resolve_round_include from .payload import _bake_overrides, _resolve_variant_payloads -from .planner import filter_plan_to_variant, plan_dev_inventory_matrix +from .planner import filter_plan_to_variants, plan_dev_inventory_matrix from .spec import DevInventorySpec, PlanEntry # Private helper names appear in `__all__` to silence ruff's F401 @@ -51,6 +51,6 @@ __all__ = [ "_resolve_variant_payloads", "build_dev_inventory", "build_dev_inventory_matrix", - "filter_plan_to_variant", + "filter_plan_to_variants", "plan_dev_inventory_matrix", ] diff --git a/cli/administration/deploy/development/inventory/planner.py b/cli/administration/deploy/development/inventory/planner.py index 2031e59b7b..a6e78a87d4 100644 --- a/cli/administration/deploy/development/inventory/planner.py +++ b/cli/administration/deploy/development/inventory/planner.py @@ -25,24 +25,28 @@ if TYPE_CHECKING: from .spec import PlanEntry -def filter_plan_to_variant( +def filter_plan_to_variants( plan: list[PlanEntry], - variant: int | None, + variants: list[int] | None, ) -> list[PlanEntry]: - """Pin a matrix plan to a single round when `variant` is set. + """Pin a matrix plan to a subset of rounds when `variants` is set. - `variant=None` returns the plan unchanged (full-matrix mode). An - explicit variant index is matched against the plan's round indices; - if it's out of range, raises `ValueError` so callers can surface a - clean operator-facing error rather than silently doing nothing. + `variants=None` returns the plan unchanged (full-matrix mode). This is the + runner-split entry point: a role with more variants than one runner should + iterate is sliced into bundles (e.g. ``0,1,2`` on one runner and ``3,4`` on + the next), each bundle passed as `variants`. Out-of-range indices raise + `ValueError` so the caller surfaces a clean operator error. """ - if variant is None: + if variants is None: return plan - for entry in plan: - if entry[0] == variant: - return [entry] available = sorted(entry[0] for entry in plan) - raise ValueError(f"variant {variant} out of range; available rounds: {available}") + missing = sorted(v for v in variants if v not in available) + if missing: + raise ValueError( + f"variants {missing} out of range; available rounds: {available}" + ) + wanted = set(variants) + return [entry for entry in plan if entry[0] in wanted] def plan_dev_inventory_matrix( diff --git a/cli/administration/deploy/development/variant_select.py b/cli/administration/deploy/development/variant_select.py new file mode 100644 index 0000000000..4b17ca7fcc --- /dev/null +++ b/cli/administration/deploy/development/variant_select.py @@ -0,0 +1,53 @@ +"""Shared variant-selection for the dev-deploy ``init`` and ``deploy`` commands. + +A single ``--variant`` (comma-separated, zero-based) selects the matrix rounds a +run iterates: ``2`` pins one round, ``0,1,2`` is the runner-split bundle from CI +discovery, empty/unset is the full matrix. One value is just a one-element +bundle โ€” there is no separate "variants" flag โ€” and whether inter-round cleanup +runs follows the number of selected rounds, not the flag. Centralising this here +keeps init and deploy walking the exact same selection (SPOT).""" + +from __future__ import annotations + +import argparse +import os + +from .inventory import filter_plan_to_variants + + +def parse_variant_csv(raw: str) -> list[int]: + try: + return [int(tok.strip()) for tok in raw.split(",") if tok.strip()] + except ValueError: + raise argparse.ArgumentTypeError( + f"must be a comma-separated list of integers, got {raw!r}" + ) from None + + +def env_variant() -> list[int] | None: + raw = os.environ.get("variant", "").strip() + if not raw: + return None + try: + return parse_variant_csv(raw) + except argparse.ArgumentTypeError as exc: + raise SystemExit(f"variant environment variable {exc}") from None + + +def add_variant_args(parser: argparse.ArgumentParser, *, action: str) -> None: + parser.add_argument( + "--variant", + type=parse_variant_csv, + default=env_variant(), + help=( + f"Pin the matrix {action} to one or more rounds (comma-separated " + "zero-based indices, e.g. `2` or `0,1,2`). A single index pins one " + "round; multiple indices are the runner-split bundle. Empty/unset = " + "full matrix. Defaults to the variant environment variable when set." + ), + ) + + +def apply_variant_filter(plan, args): + """Filter `plan` to the rounds in `--variant`; empty/None = full matrix.""" + return filter_plan_to_variants(plan, args.variant or None) diff --git a/cli/contributing/changelog/archive/changelog_md.py b/cli/contributing/changelog/archive/changelog_md.py index 9bf95bb9bc..3b4a549927 100644 --- a/cli/contributing/changelog/archive/changelog_md.py +++ b/cli/contributing/changelog/archive/changelog_md.py @@ -95,6 +95,9 @@ def trim_and_archive( ) # nocheck: cache-read โ€” function reads then rewrites changelog_path; cached value would go stale on subsequent calls entries, _existing_trailing = split_into_entries(content) + first_header = _VERSION_HEADER_RE.search(content) + preamble = content[: first_header.start()] if first_header else "" + keep_entries = entries[:keep] archive_entries = entries[keep:] archive_paths: list[Path] = [ @@ -118,9 +121,9 @@ def trim_and_archive( keep_text = "".join(body for _v, _d, body in keep_entries).rstrip() index_section = build_index_section(archive_dir, repo_root) if index_section: - new_content = keep_text + "\n\n" + index_section + new_content = preamble + keep_text + "\n\n" + index_section else: - new_content = keep_text + "\n" if keep_text else "" + new_content = preamble + keep_text + "\n" if keep_text else "" if new_content != content: changelog_path.write_text(new_content, encoding="utf-8") diff --git a/cli/contributing/changelog/archive/package_mirror.py b/cli/contributing/changelog/archive/package_mirror.py index 7b04890837..1e9ff62010 100644 --- a/cli/contributing/changelog/archive/package_mirror.py +++ b/cli/contributing/changelog/archive/package_mirror.py @@ -1,9 +1,12 @@ """Mirror the kept ``CHANGELOG.md`` entries into the package changelogs. Both targets (Debian changelog and the RPM ``%changelog`` section) are -fully regenerated on every run from the kept entries plus a trailing -notice that points at the documentation site and lists the archived -versions and dates as plain text. Because the package files are +fully regenerated on every run from the kept entries plus an +archived-releases notice that points at the documentation site and lists +the archived versions and dates as plain text. The notice is folded into +the body of the oldest kept entry (not appended after the last entry), +because free text after the final entry is not valid changelog data and +makes the native parsers warn. Because the package files are regenerated, they cannot drift from the CHANGELOG.md SPOT. """ @@ -39,21 +42,33 @@ def build_package_footer(archived: list[tuple[str, str]]) -> str: def format_debian_entry( - version: str, date_iso: str, body_md: str, maintainer: str + version: str, + date_iso: str, + body_md: str, + maintainer: str, + footer_md: str = "", ) -> str: """Render one CHANGELOG.md entry as a Debian changelog stanza. - The first non-empty line of the markdown body gets a two-space - indent so it satisfies Debian's "summary line MUST start with two - spaces and a bullet" expectation; subsequent lines stay verbatim. - The signature time is a placeholder because CHANGELOG.md only - carries calendar dates. + Every non-empty line of the markdown body gets a two-space indent so + Debian's parser accepts it: each change line MUST start with at least + two spaces, otherwise dpkg treats it as a badly formatted line, loses + the entry boundary and reports an empty maintainer. Blank lines stay + blank. + + *footer_md* (the archived-releases notice) is folded into this + entry's change body โ€” indented like the rest โ€” rather than appended + after the trailer, because free text after the final ``--`` line is + not valid changelog data and makes ``dpkg-parsechangelog`` warn. The + signature time is a placeholder because CHANGELOG.md only carries + calendar dates. """ sig_date = date.fromisoformat(date_iso).strftime("%a, %d %b %Y") body = md_body_after_header(f"## [{version}] - {date_iso}\n{body_md}") lines = body.rstrip().split("\n") - if lines: - lines[0] = " " + lines[0] + if footer_md: + lines.extend(footer_md.rstrip("\n").split("\n")) + lines = [" " + ln if ln.strip() else ln for ln in lines] body_text = "\n".join(lines) return ( f"infinito-nexus ({version}-1) {_DEBIAN_DISTRIBUTION}; " @@ -65,13 +80,28 @@ def format_debian_entry( ) -def format_rpm_entry(version: str, date_iso: str, body_md: str, maintainer: str) -> str: - """Render one CHANGELOG.md entry as an RPM ``%changelog`` stanza.""" +def format_rpm_entry( + version: str, + date_iso: str, + body_md: str, + maintainer: str, + footer_md: str = "", +) -> str: + """Render one CHANGELOG.md entry as an RPM ``%changelog`` stanza. + + Every non-empty body line gets a ``- `` continuation prefix so rpm + does not read a markdown ``*`` bullet at column 0 as a new changelog + entry header (which makes ``rpmspec`` report ``bad date``). Blank + lines stay blank. *footer_md* (the archived-releases notice) is + folded into this entry's body for the same reason it is on the Debian + side: trailing free text confuses the parser. + """ rpm_date = date.fromisoformat(date_iso).strftime("%a %b %d %Y") body = md_body_after_header(f"## [{version}] - {date_iso}\n{body_md}") lines = body.rstrip().split("\n") - if lines: - lines[0] = "- " + lines[0] + if footer_md: + lines.extend(footer_md.rstrip("\n").split("\n")) + lines = ["- " + ln if ln.strip() else ln for ln in lines] body_text = "\n".join(lines) return f"* {rpm_date} {maintainer} - {version}-1\n{body_text}\n" @@ -96,10 +126,13 @@ def mirror_to_debian_changelog( """ if not entries: return False - blocks = [format_debian_entry(v, d, b, maintainer) for v, d, b in entries] + footer = build_package_footer(archived) if archived else "" + last = len(entries) - 1 + blocks = [ + format_debian_entry(v, d, b, maintainer, footer if i == last else "") + for i, (v, d, b) in enumerate(entries) + ] new_content = "\n".join(blocks).rstrip() + "\n" - if archived: - new_content += build_package_footer(archived) if dry_run: return True if path.is_file(): @@ -138,10 +171,13 @@ def mirror_to_rpm_spec_changelog( if section is None: return False head = content[: section.end()] + "\n" - blocks = [format_rpm_entry(v, d, b, maintainer) for v, d, b in entries] + footer = build_package_footer(archived) if archived else "" + last = len(entries) - 1 + blocks = [ + format_rpm_entry(v, d, b, maintainer, footer if i == last else "") + for i, (v, d, b) in enumerate(entries) + ] new_changelog = "\n".join(blocks).rstrip() + "\n" - if archived: - new_changelog += build_package_footer(archived) new_content = head + new_changelog if dry_run: return True diff --git a/cli/meta/roles/applications/ressources/__main__.py b/cli/meta/roles/applications/ressources/__main__.py index 1462e2e0e0..d8af8ae2cb 100755 --- a/cli/meta/roles/applications/ressources/__main__.py +++ b/cli/meta/roles/applications/ressources/__main__.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 +"""CLI entrypoint: list and aggregate the compose services of a role and its +shared dependencies, with optional variant overlay, depth limit, filtering and +ordering. Collection / aggregation / query / rendering live in sibling modules.""" + from __future__ import annotations import argparse -import json import sys from typing import Any @@ -10,343 +13,57 @@ from . import PROJECT_ROOT sys.path.insert(0, str(PROJECT_ROOT)) -from humanfriendly import format_size, parse_size - +from utils.cache.applications import get_variants from utils.roles.applications.services.registry import ( build_service_registry_from_applications, load_applications_from_roles_dir, ) -from utils.roles.entity_name import get_entity_name +from utils.roles.applications.services.resources import ( + SUMMABLE_FIELDS, + aggregate, + collect_role_resources, +) + +from .query import apply_filters, apply_order +from .render import ( + DEFAULT_TOTAL_LABEL, + render_json, + render_summary_json, + render_summary_text, + render_text, +) ROLES_DIR = PROJECT_ROOT / "roles" -def _as_mapping(value: Any) -> dict[str, Any]: - return value if isinstance(value, dict) else {} - - -def _parse_mem_bytes(value: Any) -> int | None: - if value is None: +def _resolve_order(tokens: list[str] | None) -> tuple[str, str] | None: + if not tokens: return None - if isinstance(value, (int, float)): - return int(value) - text = str(value).strip() - if not text: - return None - try: - return int(parse_size(text)) - except Exception: - return None - - -def _parse_cpus(value: Any) -> float | None: - if value is None: - return None - if isinstance(value, (int, float)): - return float(value) - try: - return float(str(value).strip()) - except (TypeError, ValueError): - return None - - -def _parse_int(value: Any) -> int | None: - if value is None: - return None - if isinstance(value, bool): - return None - if isinstance(value, int): - return value - try: - return int(str(value).strip()) - except (TypeError, ValueError): - return None - - -def _is_enabled(service_conf: dict[str, Any], is_primary: bool) -> bool: - if "enabled" not in service_conf: - return is_primary - raw = service_conf.get("enabled") - if isinstance(raw, bool): - return raw - text = str(raw).strip().lower() - return text not in ("false", "0", "no", "off") - - -def _is_shared(service_conf: dict[str, Any]) -> bool: - raw = service_conf.get("shared", False) - if isinstance(raw, bool): - return raw - return str(raw).strip().lower() in ("true", "1", "yes", "on") - - -_RESOURCE_KEYS = ("mem_reservation", "mem_limit", "pids_limit", "cpus") -_CONTAINER_KEYS = ("image", "name", "version", "container") - - -def _looks_like_container(service_conf: dict[str, Any]) -> bool: - return any(key in service_conf for key in _RESOURCE_KEYS + _CONTAINER_KEYS) - - -def _row_for_service( - role_name: str, - service_key: str, - service_conf: dict[str, Any], -) -> dict[str, Any]: - return { - "role": role_name, - "service": service_key, - "mem_reservation_raw": service_conf.get("mem_reservation"), - "mem_limit_raw": service_conf.get("mem_limit"), - "pids_limit_raw": service_conf.get("pids_limit"), - "cpus_raw": service_conf.get("cpus"), - "mem_reservation_bytes": _parse_mem_bytes(service_conf.get("mem_reservation")), - "mem_limit_bytes": _parse_mem_bytes(service_conf.get("mem_limit")), - "pids_limit_int": _parse_int(service_conf.get("pids_limit")), - "cpus_float": _parse_cpus(service_conf.get("cpus")), - } - - -def collect_role_resources( - role_name: str, - applications: dict[str, dict[str, Any]], - service_registry: dict[str, dict[str, Any]], - visited: set, - rows: list[dict[str, Any]], - warnings: list[str], -) -> None: - if role_name in visited: - return - visited.add(role_name) - - if role_name not in applications: - warnings.append(f"role '{role_name}' has no meta/services.yml; skipping") - return - - config = _as_mapping(applications[role_name]) - services = _as_mapping(config.get("services")) - entity_name = get_entity_name(role_name) - - if entity_name and entity_name in services: - primary_conf = _as_mapping(services.get(entity_name)) - rows.append(_row_for_service(role_name, entity_name, primary_conf)) - else: - warnings.append( - f"role '{role_name}' has no services.{entity_name or ''} entry" - ) - - shared_dependencies: list[str] = [] - for service_key, raw_service_conf in services.items(): - if service_key == entity_name: - continue - service_conf = _as_mapping(raw_service_conf) - if not service_conf: - continue - - if not _is_enabled(service_conf, is_primary=False): - continue - - if _is_shared(service_conf): - provider = _as_mapping(service_registry.get(service_key)) - provider_role = provider.get("role") if provider else None - if provider_role and provider_role != role_name: - shared_dependencies.append(provider_role) - elif not provider_role: - warnings.append( - f"{role_name}: shared service '{service_key}' has no registered provider" - ) - else: - if not _looks_like_container(service_conf): - continue - rows.append(_row_for_service(role_name, service_key, service_conf)) - - for provider_role in shared_dependencies: - collect_role_resources( - provider_role, - applications, - service_registry, - visited, - rows, - warnings, - ) - - -def aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: - total_mem_res = 0 - total_mem_lim = 0 - total_pids = 0 - max_cpus = 0.0 - any_mem_res = any_mem_lim = any_pids = any_cpus = False - - for row in rows: - if row["mem_reservation_bytes"] is not None: - total_mem_res += row["mem_reservation_bytes"] - any_mem_res = True - if row["mem_limit_bytes"] is not None: - total_mem_lim += row["mem_limit_bytes"] - any_mem_lim = True - if row["pids_limit_int"] is not None: - total_pids += row["pids_limit_int"] - any_pids = True - if row["cpus_float"] is not None: - max_cpus = max(max_cpus, row["cpus_float"]) - any_cpus = True - - return { - "mem_reservation_bytes": total_mem_res if any_mem_res else None, - "mem_limit_bytes": total_mem_lim if any_mem_lim else None, - "pids_limit_int": total_pids if any_pids else None, - "cpus_float": max_cpus if any_cpus else None, - } - - -def _fmt_mem(value: int | None) -> str: - if value is None: - return "-" - return format_size(value, binary=False) - - -def _fmt_int(value: int | None) -> str: - return "-" if value is None else str(value) - - -def _fmt_float(value: float | None) -> str: - if value is None: - return "-" - if float(value).is_integer(): - return str(int(value)) - return f"{value:g}" - - -def render_text( - role_name: str, - rows: list[dict[str, Any]], - totals: dict[str, Any], - warnings: list[str], -) -> str: - headers = ["service", "role", "mem_reservation", "mem_limit", "pids_limit", "cpus"] - table_rows: list[tuple[str, ...]] = [] - - table_rows.extend( - ( - row["service"], - row["role"], - _fmt_mem(row["mem_reservation_bytes"]), - _fmt_mem(row["mem_limit_bytes"]), - _fmt_int(row["pids_limit_int"]), - _fmt_float(row["cpus_float"]), - ) - for row in sorted(rows, key=lambda r: (r["service"], r["role"])) - ) - - total_label = "TOTAL (mem=SUM, pids=SUM max-provisioned, cpus=MAX)" - total_row = ( - total_label, - "", - _fmt_mem(totals["mem_reservation_bytes"]), - _fmt_mem(totals["mem_limit_bytes"]), - _fmt_int(totals["pids_limit_int"]), - _fmt_float(totals["cpus_float"]), - ) - - widths = [len(h) for h in headers] - for r in [*table_rows, total_row]: - for i, cell in enumerate(r): - widths[i] = max(widths[i], len(cell)) - - def fmt_row(cells: tuple[str, ...]) -> str: - return " ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells)) - - sep = " ".join("-" * w for w in widths) - lines = [ - f"# Resources for role: {role_name}", - "", - fmt_row(tuple(headers)), - sep, - ] - lines.extend(fmt_row(r) for r in table_rows) - lines.append(sep) - lines.append(fmt_row(total_row)) - - if warnings: - lines.append("") - lines.append("# Warnings") - lines.extend(f"! {w}" for w in warnings) - - return "\n".join(lines) - - -def render_json( - role_name: str, - rows: list[dict[str, Any]], - totals: dict[str, Any], - warnings: list[str], -) -> str: - def _row(row: dict[str, Any]) -> dict[str, Any]: - return { - "role": row["role"], - "service": row["service"], - "mem_reservation": { - "raw": row["mem_reservation_raw"], - "bytes": row["mem_reservation_bytes"], - "human": _fmt_mem(row["mem_reservation_bytes"]), - }, - "mem_limit": { - "raw": row["mem_limit_raw"], - "bytes": row["mem_limit_bytes"], - "human": _fmt_mem(row["mem_limit_bytes"]), - }, - "pids_limit": { - "raw": row["pids_limit_raw"], - "value": row["pids_limit_int"], - }, - "cpus": { - "raw": row["cpus_raw"], - "value": row["cpus_float"], - }, - } - - payload = { - "role": role_name, - "services": [_row(r) for r in rows], - "totals": { - "mem_reservation": { - "bytes": totals["mem_reservation_bytes"], - "human": _fmt_mem(totals["mem_reservation_bytes"]), - }, - "mem_limit": { - "bytes": totals["mem_limit_bytes"], - "human": _fmt_mem(totals["mem_limit_bytes"]), - }, - "pids_limit": {"value": totals["pids_limit_int"]}, - "cpus": {"value": totals["cpus_float"]}, - "aggregation": { - "mem_reservation": "sum", - "mem_limit": "sum", - "pids_limit": "sum (max-provisioned; per-container cap, not shared load)", - "cpus": "max", - }, - }, - "warnings": warnings, - } - return json.dumps(payload, indent=2) + if len(tokens) == 1: + return ("asc", tokens[0]) + direction, field = tokens[0].lower(), tokens[1] + if direction not in ("asc", "desc"): + raise SystemExit(f"--order direction must be asc|desc, got '{tokens[0]}'") + return (direction, field) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( - "Compute aggregated container resources (mem_reservation, mem_limit, " - "pids_limit, cpus) for an Ansible role. Follows enabled shared services " - "recursively via the service registry. mem_reservation/mem_limit are " - "summed; pids_limit is summed as a max-provisioned host-pid budget " - "(per-container cap, not actual shared load); cpus is max (shared " - "across containers, not additive)." + "List and aggregate the compose services of an Ansible role and its " + "shared dependencies (resolved recursively via the service registry). " + "mem_reservation/mem_limit are summed, pids_limit is summed as a " + "max-provisioned host-pid budget, cpus is max." ) ) parser.add_argument( "--role", - required=True, - help="Role name (directory under roles/), e.g. web-app-peertube", + default=None, + help="Role name (directory under roles/), e.g. web-app-peertube. Three " + "modes: (1) --role without --variant: one row per variant of that role " + "(mem/pids = sum, cpus = max). (2) --role with --variant N: the detailed " + "per-service table for that variant. (3) no --role: one row per role (its " + "heaviest variant, or the --variant N footprint of every role).", ) parser.add_argument( "-f", @@ -355,17 +72,272 @@ def parse_args() -> argparse.Namespace: default="text", help="Output format (default: text).", ) + parser.add_argument( + "--order", + nargs="+", + metavar="[asc|desc] FIELD", + help="Order rows by FIELD. FIELD is one of service, role, depth, bond, " + "mem_reservation, mem_limit, min_storage, pids_limit, cpus. Direction " + "defaults to asc.", + ) + parser.add_argument( + "--filter", + metavar="EXPR", + help="Filter rows, e.g. 'bond<=0.5 & cpus>=1 & mem_limit>=512m'. Fields: " + "bond, depth, mem_reservation, mem_limit, min_storage, pids_limit, cpus. " + "Operators: <= >= < > == != ; combine with '&'.", + ) + parser.add_argument( + "--depth", + type=int, + default=0, + help="Max recursion depth over parent/shared services (0 = unlimited). " + "1 = the role's own services only.", + ) + parser.add_argument( + "--variant", + type=int, + default=None, + help="Variant index from the role's meta/variants.yml (default: base " + "config). Applies the same variant overlay as inventory creation.", + ) + parser.add_argument( + "--sum", + nargs="*", + metavar="FIELD", + default=None, + help="Show a SUM row instead of the default total. Bare --sum sums all " + "fields (mem_reservation, mem_limit, pids_limit, cpus, bond); pass field " + "names to sum only those.", + ) + parser.add_argument( + "--unshared", + action="store_true", + help="List every service occurrence individually instead of loading each " + "service only once (the default deduplicates shared services).", + ) return parser.parse_args() +def _max_over_none(values: Any) -> Any: + present = [v for v in values if v is not None] + return max(present) if present else None + + +def _aggregate_role_at( + role: str, + applications: dict[str, Any], + service_registry: dict[str, Any], + variant_conf: Any, + max_depth: int, + dedup: bool, +) -> dict[str, Any]: + """Aggregate (mem_reservation/mem_limit/pids_limit summed, cpus max) for one + role at one variant overlay (or the base config when variant_conf is None). + Collection warnings are discarded to keep the summary views readable.""" + apps = applications + if variant_conf is not None: + apps = dict(applications) + apps[role] = variant_conf or {} + rows: list[dict[str, Any]] = [] + collect_role_resources( + role_name=role, + applications=apps, + service_registry=service_registry, + visited=set(), + rows=rows, + warnings=[], + max_depth=max_depth, + dedup=dedup, + ) + return aggregate(rows) + + +def _summary_row(key_field: str, key_value: Any, agg: dict[str, Any]) -> dict[str, Any]: + return { + key_field: key_value, + "service": "", + "depth": None, + "bond_float": None, + "mem_reservation_bytes": agg.get("mem_reservation_bytes"), + "mem_limit_bytes": agg.get("mem_limit_bytes"), + "min_storage_bytes": agg.get("min_storage_bytes"), + "pids_limit_int": agg.get("pids_limit_int"), + "cpus_float": agg.get("cpus_float"), + } + + +def _role_resource_row( + role: str, + applications: dict[str, Any], + service_registry: dict[str, Any], + variants_per_app: dict[str, Any], + variant_index: int | None, + max_depth: int, + dedup: bool, +) -> dict[str, Any]: + """One per-role footprint row: the heaviest variant (max of the per-variant + sums for mem_reservation/mem_limit/pids_limit, max for cpus), or the chosen + variant's sum/max when variant_index is given.""" + app_variants = variants_per_app.get(role) or [] + + def agg_at(conf: Any) -> dict[str, Any]: + return _aggregate_role_at( + role, applications, service_registry, conf, max_depth, dedup + ) + + if variant_index is not None: + in_range = 0 <= variant_index < len(app_variants) + agg = agg_at(app_variants[variant_index] if in_range else None) + elif app_variants: + per_variant = [agg_at(conf) for conf in app_variants] + agg = { + "mem_reservation_bytes": _max_over_none( + a["mem_reservation_bytes"] for a in per_variant + ), + "mem_limit_bytes": _max_over_none( + a["mem_limit_bytes"] for a in per_variant + ), + "min_storage_bytes": _max_over_none( + a.get("min_storage_bytes") for a in per_variant + ), + "pids_limit_int": _max_over_none(a["pids_limit_int"] for a in per_variant), + "cpus_float": _max_over_none(a["cpus_float"] for a in per_variant), + } + else: + agg = agg_at(None) + + return _summary_row("role", role, agg) + + +def _run_role_variants( + args: argparse.Namespace, + order: tuple[str, str] | None, + applications: dict[str, Any], +) -> int: + variants_per_app = get_variants(roles_dir=str(ROLES_DIR)) + service_registry = build_service_registry_from_applications(applications) + app_variants = variants_per_app.get(args.role) or [] + dedup = not args.unshared + + warnings: list[str] = [] + rows: list[dict[str, Any]] = [] + if app_variants: + for idx, conf in enumerate(app_variants): + agg = _aggregate_role_at( + args.role, applications, service_registry, conf, args.depth, dedup + ) + rows.append(_summary_row("variant", idx, agg)) + else: + agg = _aggregate_role_at( + args.role, applications, service_registry, None, args.depth, dedup + ) + rows.append(_summary_row("variant", "base", agg)) + warnings.append( + f"role '{args.role}' has no meta/variants.yml; showing base config" + ) + + try: + rows = apply_filters(rows, args.filter) + except ValueError as exc: + raise SystemExit(f"--filter: {exc}") from exc + + presorted = False + if order is not None: + try: + rows = apply_order(rows, order[0], order[1]) + except ValueError as exc: + raise SystemExit(f"--order: {exc}") from exc + presorted = True + + title = f"Resource footprint per variant of {args.role} (mem/pids=sum, cpus=max)" + label = "per variant: mem/pids=sum, cpus=max" + if args.format == "json": + print(render_summary_json(rows, "variant", label, warnings)) + else: + print( + render_summary_text(rows, "variant", title, warnings, presorted=presorted) + ) + return 0 + + +def _run_all_roles( + args: argparse.Namespace, + order: tuple[str, str] | None, + applications: dict[str, Any], +) -> int: + variants_per_app = get_variants(roles_dir=str(ROLES_DIR)) + service_registry = build_service_registry_from_applications(applications) + + roles = [ + _role_resource_row( + role, + applications, + service_registry, + variants_per_app, + args.variant, + args.depth, + not args.unshared, + ) + for role in sorted(applications) + ] + + try: + roles = apply_filters(roles, args.filter) + except ValueError as exc: + raise SystemExit(f"--filter: {exc}") from exc + + presorted = False + if order is not None: + try: + roles = apply_order(roles, order[0], order[1]) + except ValueError as exc: + raise SystemExit(f"--order: {exc}") from exc + presorted = True + + if args.variant is not None: + label = f"variant {args.variant}: mem/pids=sum, cpus=max" + else: + label = "heaviest variant: mem/pids=max(variant-sum), cpus=max" + title = f"Resource footprint per role ({label})" + + if args.format == "json": + print(render_summary_json(roles, "role", label, warnings=[])) + else: + print( + render_summary_text(roles, "role", title, warnings=[], presorted=presorted) + ) + return 0 + + def main() -> int: args = parse_args() + order = _resolve_order(args.order) applications = load_applications_from_roles_dir(ROLES_DIR) + + if args.role is None: + return _run_all_roles(args, order, applications) + + if args.variant is None: + return _run_role_variants(args, order, applications) + + warnings: list[str] = [] + + if args.variant is not None: + app_variants = get_variants(roles_dir=str(ROLES_DIR)).get(args.role) or [] + if 0 <= args.variant < len(app_variants): + applications = dict(applications) + applications[args.role] = app_variants[args.variant] or {} + else: + warnings.append( + f"variant {args.variant} out of range for '{args.role}' " + f"({len(app_variants)} variant(s)); using base config" + ) + service_registry = build_service_registry_from_applications(applications) rows: list[dict[str, Any]] = [] - warnings: list[str] = [] collect_role_resources( role_name=args.role, applications=applications, @@ -373,14 +345,46 @@ def main() -> int: visited=set(), rows=rows, warnings=warnings, + max_depth=args.depth, + dedup=not args.unshared, ) - totals = aggregate(rows) + try: + rows = apply_filters(rows, args.filter) + except ValueError as exc: + raise SystemExit(f"--filter: {exc}") from exc + + try: + totals = aggregate(rows, sum_fields=args.sum) + except ValueError as exc: + raise SystemExit(f"--sum: {exc}") from exc + + if args.sum is None: + total_label = DEFAULT_TOTAL_LABEL + else: + total_label = "SUM (" + ", ".join(args.sum or sorted(SUMMABLE_FIELDS)) + ")" + + presorted = False + if order is not None: + try: + rows = apply_order(rows, order[0], order[1]) + except ValueError as exc: + raise SystemExit(f"--order: {exc}") from exc + presorted = True if args.format == "json": print(render_json(args.role, rows, totals, warnings)) else: - print(render_text(args.role, rows, totals, warnings)) + print( + render_text( + args.role, + rows, + totals, + warnings, + presorted=presorted, + total_label=total_label, + ) + ) return 0 diff --git a/cli/meta/roles/applications/ressources/query.py b/cli/meta/roles/applications/ressources/query.py new file mode 100644 index 0000000000..3819fe6d92 --- /dev/null +++ b/cli/meta/roles/applications/ressources/query.py @@ -0,0 +1,102 @@ +"""Filter and order the collected service rows by their numeric/scalar fields +(bond, depth, the resource columns) and by service/role.""" + +from __future__ import annotations + +import operator +import re +from typing import TYPE_CHECKING, Any + +from utils.roles.applications.services.resources import ( + _parse_cpus, + _parse_int, + _parse_mem_bytes, +) + +if TYPE_CHECKING: + from collections.abc import Callable + +_FIELD_ACCESSORS: dict[str, Callable[[dict[str, Any]], Any]] = { + "service": lambda r: r.get("service"), + "role": lambda r: r.get("role"), + "variant": lambda r: r.get("variant"), + "depth": lambda r: r.get("depth"), + "bond": lambda r: r.get("bond_float"), + "mem_reservation": lambda r: r.get("mem_reservation_bytes"), + "mem_limit": lambda r: r.get("mem_limit_bytes"), + "min_storage": lambda r: r.get("min_storage_bytes"), + "pids_limit": lambda r: r.get("pids_limit_int"), + "cpus": lambda r: r.get("cpus_float"), +} + +_OPS: dict[str, Callable[[Any, Any], bool]] = { + "<=": operator.le, + ">=": operator.ge, + "!=": operator.ne, + "==": operator.eq, + "=": operator.eq, + "<": operator.lt, + ">": operator.gt, +} + +_FILTER_RE = re.compile(r"^\s*([a-z_]+)\s*(<=|>=|!=|==|=|<|>)\s*(.+?)\s*$") + + +def _parse_filter_value(field: str, raw: str) -> Any: + if field in ("mem_reservation", "mem_limit", "min_storage"): + value = _parse_mem_bytes(raw) + elif field in ("pids_limit", "depth"): + value = _parse_int(raw) + else: + value = _parse_cpus(raw) + if value is None: + raise ValueError(f"invalid value {raw!r} for filter field '{field}'") + return value + + +def apply_filters(rows: list[dict[str, Any]], expr: str | None) -> list[dict[str, Any]]: + if not expr: + return rows + conditions: list[tuple[str, Callable[[Any, Any], bool], Any]] = [] + for part in expr.split("&"): + if not part.strip(): + continue + match = _FILTER_RE.match(part) + if not match: + raise ValueError(f"invalid filter expression: {part.strip()!r}") + field, op, raw_value = match.group(1), match.group(2), match.group(3) + if field not in _FIELD_ACCESSORS or field in ("service", "role"): + raise ValueError(f"unknown numeric filter field: '{field}'") + conditions.append((field, _OPS[op], _parse_filter_value(field, raw_value))) + + out: list[dict[str, Any]] = [] + for row in rows: + keep = True + for field, op_fn, value in conditions: + cell = _FIELD_ACCESSORS[field](row) + if cell is None or not op_fn(cell, value): + keep = False + break + if keep: + out.append(row) + return out + + +def _order_key(value: Any) -> tuple[int, float, str]: + if value is None: + return (1, 0.0, "") + if isinstance(value, (int, float)): + return (0, float(value), "") + return (0, 0.0, str(value)) + + +def apply_order( + rows: list[dict[str, Any]], direction: str, field: str +) -> list[dict[str, Any]]: + if field not in _FIELD_ACCESSORS: + raise ValueError(f"unknown order field: '{field}'") + accessor = _FIELD_ACCESSORS[field] + present = [r for r in rows if accessor(r) is not None] + missing = [r for r in rows if accessor(r) is None] + present.sort(key=lambda r: _order_key(accessor(r)), reverse=direction == "desc") + return present + missing diff --git a/cli/meta/roles/applications/ressources/render.py b/cli/meta/roles/applications/ressources/render.py new file mode 100644 index 0000000000..c5f5b2bbd8 --- /dev/null +++ b/cli/meta/roles/applications/ressources/render.py @@ -0,0 +1,264 @@ +"""Render the collected service rows and aggregated totals as an aligned text +table or as JSON.""" + +from __future__ import annotations + +import json +from typing import Any + +from humanfriendly import format_size + + +def _fmt_mem(value: int | None) -> str: + if value is None: + return "-" + return format_size(value, binary=False) + + +def _fmt_int(value: int | None) -> str: + return "-" if value is None else str(value) + + +def _fmt_float(value: float | None) -> str: + if value is None: + return "-" + if float(value).is_integer(): + return str(int(value)) + return f"{value:g}" + + +DEFAULT_TOTAL_LABEL = "TOTAL (mem=SUM, pids=SUM max-provisioned, cpus=MAX)" + + +def render_text( + role_name: str, + rows: list[dict[str, Any]], + totals: dict[str, Any], + warnings: list[str], + presorted: bool = False, + total_label: str = DEFAULT_TOTAL_LABEL, +) -> str: + headers = [ + "service", + "role", + "depth", + "mem_reservation", + "mem_limit", + "min_storage", + "pids_limit", + "cpus", + "bond", + ] + ordered = ( + rows if presorted else sorted(rows, key=lambda r: (r["service"], r["role"])) + ) + table_rows: list[tuple[str, ...]] = [ + ( + row["service"], + row["role"], + _fmt_int(row.get("depth")), + _fmt_mem(row["mem_reservation_bytes"]), + _fmt_mem(row["mem_limit_bytes"]), + _fmt_mem(row.get("min_storage_bytes")), + _fmt_int(row["pids_limit_int"]), + _fmt_float(row["cpus_float"]), + _fmt_float(row.get("bond_float")), + ) + for row in ordered + ] + + total_row = ( + total_label, + "", + "", + _fmt_mem(totals["mem_reservation_bytes"]), + _fmt_mem(totals["mem_limit_bytes"]), + _fmt_mem(totals.get("min_storage_bytes")), + _fmt_int(totals["pids_limit_int"]), + _fmt_float(totals["cpus_float"]), + _fmt_float(totals.get("bond_float")), + ) + + widths = [len(h) for h in headers] + for r in [*table_rows, total_row]: + for i, cell in enumerate(r): + widths[i] = max(widths[i], len(cell)) + + def fmt_row(cells: tuple[str, ...]) -> str: + return " ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells)) + + sep = " ".join("-" * w for w in widths) + lines = [ + f"# Resources for role: {role_name}", + "", + fmt_row(tuple(headers)), + sep, + ] + lines.extend(fmt_row(r) for r in table_rows) + lines.append(sep) + lines.append(fmt_row(total_row)) + + if warnings: + lines.append("") + lines.append("# Warnings") + lines.extend(f"! {w}" for w in warnings) + + return "\n".join(lines) + + +def render_json( + role_name: str, + rows: list[dict[str, Any]], + totals: dict[str, Any], + warnings: list[str], +) -> str: + def _row(row: dict[str, Any]) -> dict[str, Any]: + return { + "depth": row.get("depth"), + "role": row["role"], + "service": row["service"], + "mem_reservation": { + "raw": row["mem_reservation_raw"], + "bytes": row["mem_reservation_bytes"], + "human": _fmt_mem(row["mem_reservation_bytes"]), + }, + "mem_limit": { + "raw": row["mem_limit_raw"], + "bytes": row["mem_limit_bytes"], + "human": _fmt_mem(row["mem_limit_bytes"]), + }, + "min_storage": { + "raw": row.get("min_storage_raw"), + "bytes": row.get("min_storage_bytes"), + "human": _fmt_mem(row.get("min_storage_bytes")), + }, + "pids_limit": { + "raw": row["pids_limit_raw"], + "value": row["pids_limit_int"], + }, + "cpus": { + "raw": row["cpus_raw"], + "value": row["cpus_float"], + }, + "bond": { + "raw": row.get("bond_raw"), + "value": row.get("bond_float"), + }, + } + + payload = { + "role": role_name, + "services": [_row(r) for r in rows], + "totals": { + "mem_reservation": { + "bytes": totals["mem_reservation_bytes"], + "human": _fmt_mem(totals["mem_reservation_bytes"]), + }, + "mem_limit": { + "bytes": totals["mem_limit_bytes"], + "human": _fmt_mem(totals["mem_limit_bytes"]), + }, + "min_storage": { + "bytes": totals.get("min_storage_bytes"), + "human": _fmt_mem(totals.get("min_storage_bytes")), + }, + "pids_limit": {"value": totals["pids_limit_int"]}, + "cpus": {"value": totals["cpus_float"]}, + "bond": {"value": totals.get("bond_float")}, + "aggregation": { + "mem_reservation": "sum", + "mem_limit": "sum", + "min_storage": "sum", + "pids_limit": "sum (max-provisioned; per-container cap, not shared load)", + "cpus": "max", + }, + }, + "warnings": warnings, + } + return json.dumps(payload, indent=2) + + +def render_summary_text( + rows: list[dict[str, Any]], + key_field: str, + title: str, + warnings: list[str], + presorted: bool = False, +) -> str: + headers = [ + key_field, + "mem_reservation", + "mem_limit", + "min_storage", + "pids_limit", + "cpus", + ] + ordered = rows if presorted else sorted(rows, key=lambda r: str(r[key_field])) + table_rows: list[tuple[str, ...]] = [ + ( + str(row[key_field]), + _fmt_mem(row["mem_reservation_bytes"]), + _fmt_mem(row["mem_limit_bytes"]), + _fmt_mem(row.get("min_storage_bytes")), + _fmt_int(row["pids_limit_int"]), + _fmt_float(row["cpus_float"]), + ) + for row in ordered + ] + + widths = [len(h) for h in headers] + for r in table_rows: + for i, cell in enumerate(r): + widths[i] = max(widths[i], len(cell)) + + def fmt_row(cells: tuple[str, ...]) -> str: + return " ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells)) + + sep = " ".join("-" * w for w in widths) + lines = [ + f"# {title}", + "", + fmt_row(tuple(headers)), + sep, + ] + lines.extend(fmt_row(r) for r in table_rows) + + if warnings: + lines.append("") + lines.append("# Warnings") + lines.extend(f"! {w}" for w in warnings) + + return "\n".join(lines) + + +def render_summary_json( + rows: list[dict[str, Any]], + key_field: str, + label: str, + warnings: list[str], +) -> str: + payload = { + "aggregation": label, + "rows": [ + { + key_field: row[key_field], + "mem_reservation": { + "bytes": row["mem_reservation_bytes"], + "human": _fmt_mem(row["mem_reservation_bytes"]), + }, + "mem_limit": { + "bytes": row["mem_limit_bytes"], + "human": _fmt_mem(row["mem_limit_bytes"]), + }, + "min_storage": { + "bytes": row.get("min_storage_bytes"), + "human": _fmt_mem(row.get("min_storage_bytes")), + }, + "pids_limit": {"value": row["pids_limit_int"]}, + "cpus": {"value": row["cpus_float"]}, + } + for row in rows + ], + "warnings": warnings, + } + return json.dumps(payload, indent=2) diff --git a/compose/package-cache-frontend/cert-gen.sh b/compose/package-cache-frontend/cert-gen.sh index 5a0ca72728..318c50fdb2 100755 --- a/compose/package-cache-frontend/cert-gen.sh +++ b/compose/package-cache-frontend/cert-gen.sh @@ -33,8 +33,14 @@ cert_still_valid() { openssl x509 -in "${crt}" -noout -checkend 2592000 >/dev/null 2>&1 } +# OpenSSL 3.x rejects a trust-anchor CA that lacks keyUsage, so a CA generated +# before that extension was added must be regenerated even while still valid. +ca_has_key_usage() { + openssl x509 -in "$1" -noout -text 2>/dev/null | grep -q "Key Usage" +} + ensure_ca() { - if cert_still_valid "${CA_CRT}" && [ -s "${CA_KEY}" ]; then + if cert_still_valid "${CA_CRT}" && [ -s "${CA_KEY}" ] && ca_has_key_usage "${CA_CRT}"; then log "CA already valid: ${CA_CRT}" return 0 fi diff --git a/docs/contributing/actions/deploy.md b/docs/contributing/actions/deploy.md index fff8f3c573..2d0f35d998 100644 --- a/docs/contributing/actions/deploy.md +++ b/docs/contributing/actions/deploy.md @@ -46,19 +46,22 @@ This purges the app's containers + volumes + Ansible-managed state on the host, Only return to `mode=reinstall` when you have concrete evidence that the inventory or the host stack itself is broken (for example DNS or network failures during the deploy). -## Pinning A Single Variant ๐ŸŽฏ +## Pinning Variants (Single Or Bundle) ๐ŸŽฏ -For multi-variant roles you MAY restrict any of the `make compose-deploy` invocations above (and the dev CLI subcommands) to a single matrix round by setting `variant=`: +For multi-variant roles you MAY restrict any of the `make compose-deploy` invocations above (and the dev CLI subcommands) to one or more matrix rounds by setting `variant=` โ€” a single index pins one round, a comma-separated list iterates that subset (the runner-split bundle CI produces), and empty/unset is the full matrix: ```bash # Variant 1 baseline only (no full matrix): make compose-deploy mode=reinstall apps="" full_cycle=true variant=1 +# A 3-round bundle (one CI runner's slice): +make compose-deploy mode=reinstall apps="" full_cycle=true variant=0,1,2 + # Edit-fix-redeploy loop pinned to that variant: make compose-deploy mode=update apps="" variant=1 ``` -Pinning is sticky: when iterating with `variant=`, you MUST set it on every command in the iteration. Mixing pinned and unpinned commands silently retargets a different folder. The full semantics (single-folder mode, no inter-round cleanup, out-of-range error) live in [variants.md](../design/variants.md). +Pinning is sticky: when iterating with `variant=`, you MUST set it on every command in the iteration. Mixing pinned and unpinned commands silently retargets a different folder. The full semantics (subset selection, no inter-round cleanup when one round, out-of-range error, CI runner-split) live in [variants.md](../design/variants.md). ## Direct CLI Invocation โš™๏ธ @@ -66,7 +69,7 @@ When you need a flag the make wrappers do not expose, call the dev CLI directly: ```bash infinito administration deploy development init --inventory-dir "${INFINITO_INVENTORY_DIR}" --apps "" -infinito administration deploy development deploy --inventory-dir "${INFINITO_INVENTORY_DIR}" --apps "" [--variant ] [--debug] +infinito administration deploy development deploy --inventory-dir "${INFINITO_INVENTORY_DIR}" --apps "" [--variant ] [--debug] ``` - `--inventory-dir` is always the BASE path. The wrapper appends the `-` suffix internally for matrix folders. diff --git a/docs/contributing/actions/testing/playwright.md b/docs/contributing/actions/testing/playwright.md index 3562d207e8..270e59ee4d 100644 --- a/docs/contributing/actions/testing/playwright.md +++ b/docs/contributing/actions/testing/playwright.md @@ -58,8 +58,8 @@ Any other value, including unset or empty, keeps artefacts only when a test fail The propagation chain has four hops: 1. The host environment variable `INFINITO_PLAYWRIGHT_KEEP` (set on the host shell, by `act --env INFINITO_PLAYWRIGHT_KEEP=true`, or by the GitHub Actions repository variable forwarded into each `test-deploy-*.yml` workflow as `INFINITO_PLAYWRIGHT_KEEP: ${{ vars.INFINITO_PLAYWRIGHT_KEEP }}`). -2. `roles/test-e2e-playwright/defaults/main.yml` reads the host value into `TEST_E2E_PLAYWRIGHT_KEEP` via `lookup('env', 'INFINITO_PLAYWRIGHT_KEEP')`. Inventory MAY override the role variable per environment in `group_vars` or `host_vars`. -3. `roles/test-e2e-playwright/tasks/02_run_one.yml` forwards the resolved value into the Playwright container as `-e INFINITO_PLAYWRIGHT_KEEP={{ TEST_E2E_PLAYWRIGHT_KEEP }}`. +2. `roles/test-e2e-playwright/defaults/main.yml` reads the host value into `test_e2e_playwright_keep` via `lookup('env', 'INFINITO_PLAYWRIGHT_KEEP')`. Inventory MAY override the role variable per environment in `group_vars` or `host_vars`. +3. `roles/test-e2e-playwright/tasks/02_run_one.yml` forwards the resolved value into the Playwright container as `-e INFINITO_PLAYWRIGHT_KEEP={{ test_e2e_playwright_keep }}`. 4. `roles/test-e2e-playwright/files/playwright.config.js` switches `trace`, `screenshot`, and `video` to `on` when the container value is `true`. It otherwise leaves the failure-only defaults (`retain-on-failure` for trace and video, `only-on-failure` for screenshot). Set the variable for the contexts you need: @@ -67,7 +67,7 @@ Set the variable for the contexts you need: - **Local deploy**: prefix every `make compose-deploy` invocation with `INFINITO_PLAYWRIGHT_KEEP=true`. - **Local `act` runs**: append `--env INFINITO_PLAYWRIGHT_KEEP=true` to the `act` command. - **CI**: set the `INFINITO_PLAYWRIGHT_KEEP` repository variable. For the GitHub click path see [CI Configuration](../../tools/github/actions/configuration.md#infinito_playwright_keep-). -- **Inventory pin**: set `TEST_E2E_PLAYWRIGHT_KEEP: "true"` in `group_vars` or `host_vars` of the target environment. +- **Inventory pin**: set `test_e2e_playwright_keep: "true"` in `group_vars` or `host_vars` of the target environment. | Resolved container value | Trace | Screenshot | Video | |---|---|---|---| diff --git a/docs/contributing/actions/testing/suppression.md b/docs/contributing/actions/testing/suppression.md index 03e7e83b41..63a6ef9a76 100644 --- a/docs/contributing/actions/testing/suppression.md +++ b/docs/contributing/actions/testing/suppression.md @@ -64,6 +64,7 @@ The placement rule is per check. The catalog "Position" column uses these labels | `dynamic-flag` | same line OR comment block above key | [test_dynamic_flags.py](../../../../tests/integration/roles/meta/services/test_dynamic_flags.py) | Marks a `roles/*/meta/services.yml` flag whose value stays literal. Per-flag (same line) for databases. Block-level (comment block above) when both flags stay literal. | | `native-metrics` | comment block above | [test_native_metrics_contract.py](../../../../tests/integration/roles/meta/services/prometheus/test_native_metrics_contract.py) | Marks a role whose `services.prometheus.native_metrics.enabled` and `templates/prometheus.yml.j2` presence are intentionally inconsistent. Reserved for prepared-but-default-off scrape fragments (operator opt-in). | | `variants-static-override` | same line | [test_variants_no_static_override.py](../../../../tests/integration/roles/meta/services/test_variants_no_static_override.py) | Allows a `roles/*/meta/variants.yml` leaf override of an attribute whose base value in `meta/services.yml` is a static literal. Place on the override line in `variants.yml`. Use only when the consumer chain is provably immune to the variant-merge propagation gap; the default is to make the base value dynamic via `"{{ '' in group_names }}"`. | +| `variants-shared-false-once` | same or above | [test_variants_shared_false_once.py](../../../../tests/lint/ansible/services/test_variants_shared_false_once.py) | Exempts one `roles/*/meta/variants.yml` `services..shared: false` pin from the one-local-provider-variant limit for a service whose base `meta/services.yml` uses literal `enabled: true` and dynamic `shared`. Place on the extra `shared: false` line or service key line. Place above the service key in `meta/services.yml` to exempt the service. | | `variants-services-match` | same or above | [test_services_match.py](../../../../tests/integration/roles/meta/variants/test_services_match.py) | Allows a `roles/*/meta/variants.yml` `services.` entry to reference a service key that is NOT declared as a top-level entry in `meta/services.yml`. Reserved for resolver-only matrix hooks (typically on invokable infrastructure roles) whose variant entries pin the round's companion topology via service-key flags the resolver maps to provider roles, with no real consumer contract in the base meta. | | `variants-auth-isolation` | same or above | [test_auth_isolation.py](../../../../tests/integration/roles/meta/variants/test_auth_isolation.py) | Exempts an auth-isolated variant entry in `roles/*/meta/variants.yml` (one that enables exactly one of LDAP / OIDC / OAuth2) from the rule that every non-auth service-key in the role's variant union must be pinned to literal `false`. Place on the leading `- services:` line of the variant or on the line above. Use only when the broader closure is intentional. | | `lookup-config-path` | same or above | [config/](../../../../tests/integration/ansible/lookups/config/) | Skips a single `lookup('config', โ€ฆ)` call from path-validation passes. Use when the call resolves against runtime-only state, typically self-referential roles. | diff --git a/docs/contributing/artefact/files/github/workflow.md b/docs/contributing/artefact/files/github/workflow.md index c78809279b..abbbe0a9ed 100644 --- a/docs/contributing/artefact/files/github/workflow.md +++ b/docs/contributing/artefact/files/github/workflow.md @@ -19,7 +19,7 @@ Every workflow MUST follow the schema `"[Emoji] Category: Subject (Qualifier)"`. |---|---|---| | `๐Ÿ”„` | Update / Sync | Automated version or dependency updates | | `๐Ÿชž` | Mirror | Image mirroring between registries | -| `๐Ÿงน` | Images | Image cleanup and pruning | +| `๐Ÿงน` | Cleanup / Images | Repository cleanup, image cleanup, and pruning | | `๐Ÿณ` | Build | Docker image builds | | `๐Ÿ”` | Lint | Static analysis and linting | | `๐Ÿ”’` | Scan | Security scanning | diff --git a/docs/contributing/artefact/git/pipeline.md b/docs/contributing/artefact/git/pipeline.md index 34008c7e8c..9b04245ee8 100644 --- a/docs/contributing/artefact/git/pipeline.md +++ b/docs/contributing/artefact/git/pipeline.md @@ -14,6 +14,10 @@ Every external CI trigger MUST route through one of the entry workflows. Each en - [entry-push-latest.yml](../../../../.github/workflows/entry-push-latest.yml): calls the orchestrator for pushes on the supported branch prefixes and additionally invokes the release workflow on version tags. For the contributor-facing release procedure see [release.md](../../actions/release.md). - [entry-manual.yml](../../../../.github/workflows/entry-manual.yml): dispatches the orchestrator manually for a chosen distro set and whitelist. +The push entry synchronizes the current repository's `main` branch from the configured source repository before CI policy and deploy discovery run. +The default source is `infinito-nexus/core`, and `CI_SYNC_MAIN_SOURCE_REPOSITORY` MAY override or disable the sync as documented in [configuration.md](../../tools/github/actions/configuration.md). +This sync **force-overwrites** the local `main`, so on a synced repository you MUST NOT work directly in `main`; use a `feature/**`, `fix/**`, or `hotfix/**` branch and treat `main` as a read-only mirror. + ## PR Scope Detection ๐Ÿ”Ž Before the full pipeline runs, CI detects the scope of the PR based on the changed files and the branch prefix (see [branch.md](branch.md)). @@ -28,7 +32,7 @@ The orchestrator runs the following stages in order: ### 1. Fork Prerequisites โณ -For fork PRs, CI waits for privileged images to be built by the `pull_request_target` event before proceeding. This ensures untrusted code never runs with elevated permissions. +For fork PRs, CI waits for privileged images to be built by the `pull_request_target` event before proceeding. This ensures untrusted code never runs with elevated permissions. A maintainer-applied `trusted-pr` label opts the PR out of this wait โ€” see [Trusted fork PRs](#trusted-fork-prs-). ### 2. Security ๐Ÿ›ก๏ธ @@ -105,3 +109,21 @@ See [configuration.md](../../tools/github/actions/configuration.md) for all repo ## Fork PRs ๐Ÿด Fork PRs run under the `pull_request` event without write permissions. Privileged steps (image builds, package writes) run via `pull_request_target` after a maintainer review. The fork CI waits for those privileged images before proceeding. See [pull-request.md](pull-request.md) for the contributor-facing fork workflow. + +By default the privileged `pull_request_target` build checks out the **base merge ref** (`refs/pull//merge`) from the base repository โ€” the PR's changes merged onto base, with all workflow and build-orchestration scripts coming from base, never from the fork. + +### Trusted fork PRs ๐Ÿ”“ + +When a maintainer applies the `trusted-pr` label, the fork PR is treated as deliberately released code: + +- The orchestrator's `wait-fork-prereq-run` job **skips the wait** for the privileged fork images. +- The privileged `pull_request_target` build switches its **build source** from the base merge ref to the **raw PR head** (`head.repo.full_name` @ `head.sha`). +- The built image **keeps the merge-SHA tag** (`ci-`) so the `pull_request` orchestrator finds it unchanged โ€” no consumer workflow has to know about the label. + +Security gates: + +- Only the `trusted-pr` label flips the build source; without it the safe base merge ref is used. Restrict who may set the label via repository label permissions / branch protection so only maintainers can grant it. +- Build-orchestration scripts (`scripts/image/push.sh` and the helper checkout) always come from the base repo at `github.sha`; only the Docker build **context** (`_src`) is the fork head. `persist-credentials: false` keeps the token out of the source checkout, and `USE_NIX_TOKEN` stays disabled for fork-sourced builds. +- Fork `pull_request` runs always receive a read-only token (GitHub-enforced); the label never grants write tokens or secrets to the untrusted event. + +Both events carry a `labeled` trigger, so applying the `trusted-pr` label starts fresh runs that already observe it: the `pull_request_target` build switches to the PR head and the `pull_request` orchestrator skips the wait โ€” no extra push is needed. (Re-running an existing run does not pick up a new label, because GitHub replays the original event payload.) The trade-off is that any label change starts a fresh PR CI run. diff --git a/docs/contributing/design/inventory.md b/docs/contributing/design/inventory.md index 4f47f5a3ce..8214d474ab 100644 --- a/docs/contributing/design/inventory.md +++ b/docs/contributing/design/inventory.md @@ -64,7 +64,7 @@ The matrix-variant planner (see [variants.md](variants.md)) calls the expansion ## Worked example: WordPress and Discourse ๐Ÿงช -The wp-discourse plugin install in [roles/web-app-wordpress/tasks/plugins/wp-discourse.yml](../../../roles/web-app-wordpress/tasks/plugins/wp-discourse.yml) does `container exec discourse rake api_key:create_master[...]` during the WordPress play. The Discourse container therefore has to exist by the time WordPress runs. +The wp-discourse plugin install in [roles/web-app-wordpress/tasks/addons/wp-discourse.yml](../../../roles/web-app-wordpress/tasks/addons/wp-discourse.yml) does `container exec discourse rake api_key:create_master[...]` during the WordPress play. The Discourse container therefore has to exist by the time WordPress runs. The wiring that makes this work is purely in the `services` declaration, not in `run_after`: diff --git a/docs/contributing/design/role/services/layout.md b/docs/contributing/design/role/services/layout.md index 9369411232..194534cfbc 100644 --- a/docs/contributing/design/role/services/layout.md +++ b/docs/contributing/design/role/services/layout.md @@ -12,6 +12,7 @@ All role-owned metadata lives under `roles//meta/.yml`. | `meta/server.yml` | CSP, `domains`, `status_codes`, plus per-role `networks.local.{subnet,dns_resolver}`. File root IS `applications..server`. | | `meta/rbac.yml` | RBAC declarations. File root IS `applications..rbac`. | | `meta/volumes.yml` | Compose volumes. File root IS the volumes map keyed by volume name. No `compose:` and no `volumes:` wrapper. | +| `meta/addons/.yml` | Optional. One file per addon (file root IS the addon spec; filename stem = addon id). See [Unified Addons](#unified-addons-metaaddons-) below. | | `meta/users.yml` | Role-local user definitions. File root IS the users map (no `users:` wrapper). | | `meta/schema.yml` | Credential schema definitions and runtime credential values. | | `meta/info.yml` | Optional. Descriptive role-level metadata (`logo`, `homepage`, `video`, `display`). File root IS `applications..info` (no `info:` wrapper). | @@ -47,14 +48,17 @@ The paths are: All non-compose top-level keys (everything except `compose:`, `server:`, `rbac:`, and `credentials:`) MUST be inlined into `meta/services.yml` under `.`, where `` is the value returned by `get_entity_name(role_name)`. -Inlined keys observed today (non-exhaustive): `plugins`, `plugins_enabled`, `email`, `ldap`, `accounts`, `scopes`, `alerting`, `addons`, `languages`, `company`, `default_quota`, `legacy_login_mask`, `site_name`, `token`, `modules`, `network`, `performance`, `preload_models`, `provision`, `features`. +Inlined keys observed today (non-exhaustive): `plugins`, `plugins_enabled`, `email`, `ldap`, `accounts`, `scopes`, `alerting`, `languages`, `company`, `default_quota`, `legacy_login_mask`, `site_name`, `token`, `modules`, `network`, `performance`, `preload_models`, `provision`, `features`. + +`addons` is **not inlined** under the primary entity: role-level extensions are a first-class, per-file topic under `meta/addons/` (see [Unified Addons](#unified-addons-metaaddons-)). +The same applies to a role that spells the concept `plugins`/`extensions`/`modules`/`mu_plugins`: those declarations live under `meta/addons/`. `compose.volumes:` is **not** inlined into services. -It moves to its own `meta/volumes.yml` (volumes are role-wide, not per-service). +It lives in its own `meta/volumes.yml` (volumes are role-wide, not per-service). ### Worked Example: `web-app-matomo` -`get_entity_name('web-app-matomo') == 'matomo'`, so every former top-level `config/main.yml` key (`site_name`, `performance`, โ€ฆ) is inlined under `matomo.`: +`get_entity_name('web-app-matomo') == 'matomo'`, so every non-compose top-level key (`site_name`, `performance`, โ€ฆ) is inlined under `matomo.`: ```yaml # roles/web-app-matomo/meta/services.yml (file root IS the services map) @@ -210,7 +214,7 @@ view: ### Port Bands ๐Ÿ“Š -The per-category port ranges that the suggester proposes from and that the lint check enforces live as a single `PORT_BANDS` map in [group_vars/all/08_networks.yml](../../../../../group_vars/all/08_networks.yml). +The per-category port ranges that the suggester proposes from and that the lint check enforces live as a single `PORT_BANDS` map in [08_networks.yml](../../../../../group_vars/all/08_networks.yml). Suggesters and lint pick up new entries automatically, with no second registration step. See `cli contributing network ports suggest` in [port.md](../../../tools/network/port.md). @@ -302,6 +306,89 @@ applications..info.{logo,homepage,video,display} The dashboard's `web-app-dashboard/lookup_plugins/docker_cards.py` reads `logo.class` and `display` from this location, while `description` and `galaxy_tags` continue to come from `galaxy_info` (Galaxy-spec fields). +## Unified Addons: `meta/addons/` ๐Ÿงฉ + +Role-level extensions, whatever a given app calls them natively (`addon`, `plugin`, `mu_plugin`, `extension`, `module`, or a network/appservice `bridge`), are declared through one unified contract under `meta/addons/` (requirement 026). +The per-app spelling under the primary service entity (`addons` / `plugins` / `modules` / `mu_plugins`) maps onto this single contract. + +Each addon lives in its own file `meta/addons/.yml`: **the file root IS the addon spec, there is NO wrapping `:` key, and the filename stem supplies the addon id.** +All files in the directory are collected into `applications..addons`, keyed by ``. +The materialised path is `applications..addons.`, read via `lookup('config', application_id, 'addons.')`. + +```yaml +# roles/web-app-friendica/meta/addons/ldapauth.yml (file root IS the addon spec) +enabled: "{{ lookup('config', application_id, 'services.ldap.enabled') | bool }}" +required: false # true for core components that must always install +mechanism: addon # addon | plugin | mu_plugin | extension | module | bridge +source: upstream # upstream | bundled | vendored | built +bridges: # optional; in-repo service keys declared in meta/services.yml + - ldap +version: "" # optional upstream pin; "" tracks the app default +group: optional # optional grouping label (e.g. odoo core/optional) +update: + monitored: true # optional; external tests check latest versions + catalog: friendica-addons # optional; supported upstream catalog adapter + upstream_id: ldapauth # optional; defaults to the addon id +config: {} # optional, opaque, role-interpreted runtime payload +``` + +### Fields + +| Field | Required | Type | Default | Notes | +|-------|----------|------|---------|-------| +| `mechanism` | yes | enum | โ€” | `addon` / `plugin` / `mu_plugin` / `extension` / `module` / `bridge`. Selects the install path. `bridge` denotes a network/appservice bridge addon (distinct from the `bridges:` field). | +| `source` | yes | enum | โ€” | `upstream` / `bundled` / `vendored` / `built`. | +| `enabled` | no | bool \| Jinja | `false` (or `true` when `required: true`) | Normalised by the loader. When the addon bridges exactly one service, reference that service's `enabled` flag instead of re-deriving group membership. | +| `required` | no | bool | `false` | `true` = baseline install contract: always installed, MAY omit `enabled`, MUST NOT set `enabled: false`. Also gates install-failure: a failed `required: true` addon hard-fails the play; a failed `required: false` addon warns, is skipped, and the play continues. | +| `bridges` | no | list | โ€” | Non-empty list of in-repo service keys; each MUST resolve to a service block in the same role's `meta/services.yml`. Omit when there is no cross-role dependency. | +| `version` | no | string | `""` | A pin MUST be a quoted string, never an unquoted number. `""` tracks the app default. | +| `group` | no | string | โ€” | Free grouping label (e.g. Odoo `core`/`optional`). MUST NOT affect enablement. | +| `update` | no | map | โ€” | `monitored` (bool, default `false`), `catalog` (a supported adapter), `upstream_id` (defaults to the addon id). | +| `config` | no | map | โ€” | Opaque, role-interpreted runtime payload. Lint does not constrain its inner shape beyond requiring a mapping. Secrets inside `config` MUST resolve through `lookup('config', application_id, 'credentials.')` and MUST NOT be inlined literally. | + +Any credential an addon needs is declared in `meta/schema.yml` `credentials:` +and read via `lookup('config', application_id, 'credentials.')`. There +is no new secret store. + +### Bridges: in-repo dependency vs network bridge + +Two distinct meanings of "bridge" MUST NOT be conflated: + +- the **`bridges:` field** names an in-repo cross-role service dependency + (e.g. an addon that talks to `svc-db-openldap` via the role's `ldap` + service block). Each listed key MUST be a service block in the same role's + `meta/services.yml`; lint fails otherwise. +- **`mechanism: bridge`** marks an addon that *is* a network/appservice + bridge to an external system (e.g. a Matrix `mautrix` bridge to WhatsApp). + +A single addon MAY be both: a `mechanism: bridge` addon MAY also declare a +`bridges:` dependency on an in-repo service it relies on. + +Front-door auth gates (e.g. an oauth2-proxy `sso` service in front of the +vhost) are NOT addon bridges unless the addon itself talks to that service; +they stay in `meta/services.yml` and MUST NOT appear under `bridges:`. + +### Loader, lint, and the materialised path + +- The application loader (`utils/cache/applications.py`) reads every + `meta/addons/.yml` like the other file-rooted meta topics and + **normalises the enable state**: + `required: true` defaults to enabled, an optional addon defaults to + disabled, and an explicit `enabled` value is preserved verbatim. No new + repo-wide dictionary is introduced. +- Schema validation lives in + [`tests/lint/ansible/services/test_addons_schema.py`](../../../../../tests/lint/ansible/services/test_addons_schema.py) + and bridge resolution / parity in + [`test_addons_bridges.py`](../../../../../tests/lint/ansible/services/test_addons_bridges.py). + Suppress with `# nocheck: addon-schema` / `# nocheck: addon-bridge` (comment + block above the addon key) or `# nocheck: addon-secret` (on a `config` leaf). +- Optional (`required: false`) addons express their enabled/disabled split as + a [`meta/variants.yml`](#) axis so the CI matrix exercises both states. + +> The Ansible-level `plugins/` directory (filters, lookups, modules) is a +> different concept from the application-level `addons` topic. See the +> [plugins README](../../../../../plugins/README.md) for the distinction. + ## Related Pages ๐Ÿ“š - [base.md](base.md) covers the service registration, loading, and injection model. diff --git a/docs/contributing/design/variants.md b/docs/contributing/design/variants.md index be77fa7f25..2fc5906474 100644 --- a/docs/contributing/design/variants.md +++ b/docs/contributing/design/variants.md @@ -16,7 +16,8 @@ The development deploy CLI uses a **folder-per-round model** that splits cleanly 4. **Deploy iteration.** [deploy.py](../../../cli/administration/deploy/development/deploy.py) re-derives the same plan via `plan_dev_inventory_matrix` (the planner is a pure function shared between init and deploy) and runs one deploy per folder. **Every round deploys its full variant-aware include set.** Round R starts from a clean host (the previous round's include is purged before round R begins, see point 5), so every dep that round R needs MUST be redeployed. 5. **Cleanup between rounds.** Before each non-first iteration the wrapper runs [entity.sh](../../../scripts/tests/deploy/local/purge/entity.sh) for **every** app in the previous round's include set. The next round therefore boots from an empty host, which guarantees cross-round state coherence: nothing the previous round deployed (whether it stays in the current round's closure or falls out) can leak into round R, so variants are free to disagree on which deps they pull in. The final round is followed by no purge so the last state remains available for inspection or follow-up specs. 6. **`--full-cycle` (async update pass).** When set (or `full_cycle=true` on `make compose-deploy`), the wrapper runs each round's deploy TWICE: first the regular sync pass, then immediately a second pass with `-e ASYNC_ENABLED=true` overriding the host_var, against the SAME variant's folder, BEFORE moving to the next round. The two passes are therefore always co-located on the same host state; the async pass never accidentally targets a host that a previous round's variant left behind. -7. **Pinning to a single round.** Both init and deploy accept `--variant ` (same semantics as the `variant=` make arg) to pin operations to a single round's folder. In single-round mode no inter-round cleanup runs, since there is no previous round to diff against. The `make compose-deploy mode=update` flows honour the same `variant=` arg by suffixing `INFINITO_INVENTORY_DIR` / `INFINITO_INVENTORY_FILE` to `-`, so a redeploy can target one specific variant without iterating the full matrix. +7. **Pinning rounds (single or bundle).** Both init and deploy accept `--variant ` (same semantics as the `variant=` make arg): a single index (`1`) pins one round, a comma-separated list (`0,1,2`) iterates that subset, and empty/unset is the full matrix. When the selection is a single round no inter-round cleanup runs (no previous round to diff against); out-of-range indices raise a clean error. The `make compose-deploy mode=update` flows honour the same `variant=` arg by suffixing `INFINITO_INVENTORY_DIR` / `INFINITO_INVENTORY_FILE` to `-` (single index), so a redeploy can target one specific variant without iterating the full matrix. +8. **CI runner-split.** The deploy-matrix discovery ([variant_bundles.py](../../../utils/github/variant_bundles.py), wired via [output_apps.sh](../../../scripts/github/resolve/output_apps.sh)) splits any role with more than `INFINITO_VARIANT_BUNDLE_SIZE` (default 3) variants into bundles of consecutive indices โ€” one matrix entry (runner) per bundle, e.g. a 5-variant role becomes `0,1,2` + `3,4`. Each runner receives its slice through the `variant` env (consumed by `--variant`) and runs every variant in its bundle to completion (no time-based cutting). The in-play loader (`utils.cache.applications.get_merged_applications`) uses each role's variant-free base config (the assembled `meta/.yml` payload) as the default and lets the inventory's `applications.` overrides win via deep merge: the inventory itself is the source of truth for what variant a round runs against. Variant 0 enumerates every dynamic service-key from `meta/services.yml` with `enabled: true, shared: true` so the "all flags on" baseline is explicit and deep-merge with a non-baseline override yields the expected variant-N shape (see [variants.md](../artefact/files/role/variants.md) for the file-format rule and [test_non_baseline_explicit_disables.py](../../../tests/integration/roles/meta/variants/test_non_baseline_explicit_disables.py) for the lint guard). diff --git a/docs/contributing/tools/github/actions/configuration.md b/docs/contributing/tools/github/actions/configuration.md index 8eb7ecb425..1932c5f36d 100644 --- a/docs/contributing/tools/github/actions/configuration.md +++ b/docs/contributing/tools/github/actions/configuration.md @@ -9,6 +9,7 @@ Repository variables are set under **Settings โ†’ Secrets and variables โ†’ Acti | Variable | Workflow | Default (unset) | Set to activate | |---|---|---|---| | `CI_CANCEL_IN_PROGRESS` | [entry-push-latest.yml](../../../../../.github/workflows/entry-push-latest.yml) | Cancels in-progress runs on new push | `false` to keep in-progress runs alive | +| `CI_SYNC_MAIN_SOURCE_REPOSITORY` | [entry-push-latest.yml](../../../../../.github/workflows/entry-push-latest.yml) | Syncs `main` from `infinito-nexus/core` before CI scope discovery | `/` to use another source, or `false`, empty, or the current repository to skip | | `CI_RUN_ON_MAIN` | [entry-push-latest.yml](../../../../../.github/workflows/entry-push-latest.yml) | Pushes to `main` skip CI | `true` to run CI on `main` pushes too | | `CI_ENABLE_AUTO_UPDATES` | [update.yml](../../../../../.github/workflows/update.yml), [dependabot-close.yml](../../../../../.github/workflows/dependabot-close.yml) | Update jobs skipped; Dependabot PRs auto-closed | `true` to allow update PRs (workflow-driven and Dependabot) | | `INFINITO_PLAYWRIGHT_KEEP` | [test-deploy-server.yml](../../../../../.github/workflows/test-deploy-server.yml), [test-deploy-universal.yml](../../../../../.github/workflows/test-deploy-universal.yml), [test-deploy-workstation.yml](../../../../../.github/workflows/test-deploy-workstation.yml), [test-deploy-local.yml](../../../../../.github/workflows/test-deploy-local.yml) | Playwright keeps trace, screenshot and video only when a test fails | `true` to keep them for every test (passing runs included) | @@ -45,6 +46,40 @@ cancel-in-progress: ${{ vars.CI_CANCEL_IN_PROGRESS != 'false' }} | `false` | `'false' != 'false'` โ†’ `false` | Does **not** cancel โœ“ | | `true` | `'true' != 'false'` โ†’ `true` | Cancels in-progress runs โœ“ | +## `CI_SYNC_MAIN_SOURCE_REPOSITORY` ๐Ÿ”„ + +Controls whether the push workflow updates the current repository's `main` branch before any later job derives deploy scope from `origin/main`. +This keeps fork branch pushes aligned with the canonical default branch so diff-driven role discovery sees only the branch's own changes. + +> โš ๏ธ **Forced overwrite โ€” do not work in `main`.** +> When the sync runs, the current repository's `main` is **force-pushed** from the source repository's `main`. Any commit that exists only on the fork's `main` is **discarded without warning** on the next push to any covered branch. +> Never commit or merge work directly into `main` on a synced repository. Always work on a `feature/**`, `fix/**`, or `hotfix/**` branch; treat `main` as a read-only mirror of the source. +> If you must use `main` as a working branch, disable the sync first (see **To disable the sync** below). + +**Default behaviour (variable not set):** +The workflow uses `infinito-nexus/core` as the source repository. +When the current repository is already `infinito-nexus/core`, the same-repository skip applies. + +**To use a different source repository:** + +1. Open the repository on GitHub. +2. Go to **Settings โ†’ Secrets and variables โ†’ Actions**. +3. Switch to the **Variables** tab. +4. Click **New repository variable**. +5. Set **Name** to `CI_SYNC_MAIN_SOURCE_REPOSITORY` and **Value** to `/`. +6. Save. + +**To disable the sync:** + +Set the variable to `false`, `0`, `no`, `off`, `none`, an empty value, or the current repository name. + +| Variable value | Behaviour | +|---|---| +| *(not set)* | Uses `infinito-nexus/core`; syncs forks and skips in `infinito-nexus/core` itself โœ“ | +| `infinito-nexus/core` in a fork | Syncs the fork's `main` from `infinito-nexus/core` โœ“ | +| current repository | Sync skipped โœ“ | +| `false`, `0`, `no`, `off`, `none`, or empty | Sync skipped โœ“ | + ## `CI_RUN_ON_MAIN` ๐ŸŽฏ Controls whether pushes to `main` trigger the CI pipeline. Pushes to all other branches covered by the workflow (`feature/**`, `hotfix/**`, `fix/**`, `alert-autofix-*`) are unaffected. diff --git a/docs/contributing/tools/github/actions/workflows.md b/docs/contributing/tools/github/actions/workflows.md index 754e715c0e..93b480eaec 100644 --- a/docs/contributing/tools/github/actions/workflows.md +++ b/docs/contributing/tools/github/actions/workflows.md @@ -25,8 +25,8 @@ Trigger column legend: **auto** = fires automatically (push, pull_request, sched | Workflow | Description | Trigger | Inputs | |---|---|---|---| -| [entry-pull-request-change.yml](../../../../../.github/workflows/entry-pull-request-change.yml): `โšก CI: Pull Request` | Detects PR scope from changed files + branch prefix, then conditionally calls the orchestrator. | auto (`pull_request`, `pull_request_target`: opened, synchronize, reopened, ready_for_review) | none | -| [entry-push-latest.yml](../../../../../.github/workflows/entry-push-latest.yml): `โšก CI: Push` | Runs the orchestrator on pushes to `main`, `feature/**`, `hotfix/**`, `fix/**`, `chore/**`, `alert-autofix-*`; on tagged `main` it also calls `release-version.yml`. Respects the `CI_CANCEL_IN_PROGRESS` and `CI_RUN_ON_MAIN` repository variables, see [configuration.md](configuration.md). | auto (`push`) | none | +| [entry-pull-request-change.yml](../../../../../.github/workflows/entry-pull-request-change.yml): `โšก CI: Pull Request` | Detects PR scope from changed files + branch prefix, then conditionally calls the orchestrator. A maintainer `trusted-pr` label builds the privileged images from the PR head โ€” see [pipeline.md](../../../artefact/git/pipeline.md#trusted-fork-prs-). | auto (`pull_request` & `pull_request_target`: opened, synchronize, reopened, ready_for_review, labeled) | none | +| [entry-push-latest.yml](../../../../../.github/workflows/entry-push-latest.yml): `โšก CI: Push` | Syncs `main` from the configured source repository, then runs the orchestrator on pushes to `main`, `feature/**`, `hotfix/**`, `fix/**`, `chore/**`, `alert-autofix-*`; on tagged `main` it also calls `release-version.yml`. Respects the `CI_CANCEL_IN_PROGRESS`, `CI_SYNC_MAIN_SOURCE_REPOSITORY`, and `CI_RUN_ON_MAIN` repository variables, see [configuration.md](configuration.md). | auto (`push`) | none | | [entry-manual.yml](../../../../../.github/workflows/entry-manual.yml): `โšก CI: Manual` | Manual dispatch of the full orchestrator for a chosen distro set and app whitelist. The `whitelist` input accepts the `__ALL__` sentinel for forced full deploy; see the "Diff-driven app selection" subsection in [pipeline.md](../../../artefact/git/pipeline.md). | manual | `distros` (default `debian`), `whitelist` (optional; `__ALL__` forces full deploy, empty triggers diff derivation, any other value is verbatim) | | [entry-pull-request-cancel.yml](../../../../../.github/workflows/entry-pull-request-cancel.yml): `๐Ÿšซ Cancel: PR Runs on Close` | Cancels active workflow runs for a PR when it is closed or converted to draft. | auto (`pull_request_target`: closed, converted_to_draft) | none | | [delete-branch.yml](../../../../../.github/workflows/delete-branch.yml): `๐Ÿšซ Cancel: Runs on Branch Delete` | Enters the push-entry concurrency group for the deleted branch so that any in-flight run on it is cancelled; a fallback step cancels remaining runs via the API. | auto (`delete`) | none | @@ -46,6 +46,7 @@ Trigger column legend: **auto** = fires automatically (push, pull_request, sched | [lint-docker.yml](../../../../../.github/workflows/lint-docker.yml): `๐Ÿ” Lint: Dockerfiles` | hadolint on `Dockerfile` with SARIF upload. | reusable | none | | [lint-python.yml](../../../../../.github/workflows/lint-python.yml): `๐Ÿ” Lint: Python` | `ruff` over the Python sources. | reusable | none | | [lint-shell.yml](../../../../../.github/workflows/lint-shell.yml): `๐Ÿ” Lint: Shell Scripts` | `shellcheck` over every `*.sh` file. | reusable | none | +| [lint-packages.yml](../../../../../.github/workflows/lint-packages.yml): `๐Ÿ” Lint: Packages` | Validates the generated distro packaging metadata (`debian/changelog` via `dpkg-parsechangelog`, the Fedora spec via `rpmspec`, the Arch `PKGBUILD` via `bash -n`) so a malformed changelog is caught before the package build. | reusable | none | ### CI images ๐Ÿณ @@ -90,6 +91,7 @@ Trigger column legend: **auto** = fires automatically (push, pull_request, sched |---|---|---|---| | [release-version.yml](../../../../../.github/workflows/release-version.yml): `๐Ÿš€ Release: Version Logic` | Releases a specific version tag (build + publish). Called from `entry-push-latest.yml` on tagged pushes. | reusable, manual | `tag` (required, e.g. `v1.2.3`) | | [release-highest.yml](../../../../../.github/workflows/release-highest.yml): `๐Ÿš€ Release: Highest Missing Version` | Scheduled backfill: finds the highest tag without a release and triggers `release-version.yml` for it. | auto (`schedule`: daily 00:00 UTC), manual | none | +| [cleanup-stale.yml](../../../../../.github/workflows/cleanup-stale.yml): `๐Ÿงน Cleanup: Stale Repository Data` | Marks and closes stale issues and PRs, deletes inactive branches, and prunes old GHCR CI image versions. | auto (`schedule`: daily 00:00 UTC), manual | none | | [update.yml](../../../../../.github/workflows/update.yml): `๐Ÿ”„ Update: Versions` | Updates Docker image versions and other pinned dependencies; opens an update PR via a GitHub App installation token so PR-lifecycle events fire on the resulting PR. Both jobs are gated behind the `CI_ENABLE_AUTO_UPDATES` repository variable, see [configuration.md](configuration.md). Requires the `BOT_APP_CLIENT_ID` and `BOT_APP_PRIVATE_KEY` repository secrets, see [secrets.md](secrets.md). | auto (`push` to `main`, `schedule`: daily 00:30 UTC), manual | none | | [dependabot-close.yml](../../../../../.github/workflows/dependabot-close.yml): `๐Ÿšซ Dependabot: Close while auto-updates disabled` | Auto-closes Dependabot PRs while `CI_ENABLE_AUTO_UPDATES` is not set to `true`, so Dependabot honours the same gate as the workflow-driven update jobs. See [configuration.md](configuration.md). | auto (`pull_request_target`: opened, reopened) | none | diff --git a/docs/requirements/026-unified-addon-syntax.md b/docs/requirements/026-unified-addon-syntax.md index 3d6b8cfa61..803163af79 100644 --- a/docs/requirements/026-unified-addon-syntax.md +++ b/docs/requirements/026-unified-addon-syntax.md @@ -2,7 +2,7 @@ ## User Story -As a contributor to Infinito.Nexus, I want every role that ships application-level extensions to declare them through one unified `meta/addons.yml` contract, instead of today's per-app spelling (`addons`, `plugins`, `extensions`, `modules`, `mu_plugins`) scattered across `meta/services.yml`, `vars/main.yml`, and `tasks/`, so that addons are discoverable, lintable, and testable with a single schema, and so that any addon that bridges into another role declares that dependency explicitly through a service flag in `meta/services.yml`. +As a contributor to Infinito.Nexus, I want every role that ships application-level extensions to declare them through one unified `meta/addons/` contract (one file per addon), instead of today's per-app spelling (`addons`, `plugins`, `extensions`, `modules`, `mu_plugins`) scattered across `meta/services.yml`, `vars/main.yml`, and `tasks/`, so that addons are discoverable, lintable, and testable with a single schema, and so that any addon that bridges into another role declares that dependency explicitly through a service flag in `meta/services.yml`. ## Background @@ -10,14 +10,14 @@ The repository already deploys role-level extension units, but each role spells - [web-app-friendica](../../roles/web-app-friendica/) calls them `addons` and inlines them under `friendica.addons` in [meta/services.yml](../../roles/web-app-friendica/meta/services.yml). - [web-app-odoo](../../roles/web-app-odoo/) calls them `modules` (split `core` / `optional`) under `odoo.modules` in [meta/services.yml](../../roles/web-app-odoo/meta/services.yml). -- [web-app-nextcloud](../../roles/web-app-nextcloud/) calls them `plugins` and keeps one file per plugin under [vars/plugins/](../../roles/web-app-nextcloud/vars/plugins/). -- [web-app-wordpress](../../roles/web-app-wordpress/) calls them `plugins` (and separately `mu_plugins`) under [tasks/plugins/](../../roles/web-app-wordpress/tasks/plugins/). +- [web-app-nextcloud](../../roles/web-app-nextcloud/) calls them `plugins` and keeps one file per plugin under `vars/plugins/`. +- [web-app-wordpress](../../roles/web-app-wordpress/) calls them `plugins` (and separately `mu_plugins`) under `tasks/plugins/`. - [web-app-mediawiki](../../roles/web-app-mediawiki/) calls them `extensions` and hard-codes the list in [vars/main.yml](../../roles/web-app-mediawiki/vars/main.yml). - [web-app-xwiki](../../roles/web-app-xwiki/) calls them `plugins` (with nested `items[].id` + `version`) under `xwiki.plugins` in [meta/services.yml](../../roles/web-app-xwiki/meta/services.yml). - [web-app-joomla](../../roles/web-app-joomla/) builds a single OIDC `plugin` in [tasks/07_oidc_plugin.yml](../../roles/web-app-joomla/tasks/07_oidc_plugin.yml). - [desk-gnome-extensions](../../roles/desk-gnome-extensions/) calls them `plugins`/`extensions` and loops over `services.gnome-extensions.plugins`. -The [per-role meta layout](../contributing/design/role/services/layout.md) already lists `addons`, `plugins`, and `modules` among the keys *inlined* under the primary service entity. This requirement promotes that concept to a first-class, file-rooted `meta/addons.yml` topic, the same move requirement 011 made for `meta/info.yml` and requirement 008/009 made for other meta topics. +The [per-role meta layout](../contributing/design/role/services/layout.md) already lists `addons`, `plugins`, and `modules` among the keys *inlined* under the primary service entity. This requirement promotes that concept to a first-class, per-file `meta/addons/.yml` topic (one file per addon, file root IS the addon spec), the same move requirement 011 made for `meta/info.yml` and requirement 008/009 made for other meta topics. A recurring property of these extensions is that many of them exist **only to bridge into another role**: `ldapauth` and `user_ldap` bridge to `svc-db-openldap`; `sociallogin`, `oidc-authenticator`, `OpenIDConnect`, `plg_system_keycloak`, and the WordPress OIDC plugin bridge to the SSO provider (`web-app-keycloak`); `wp-discourse` bridges to `web-app-discourse`; XWiki's `matomo` plugin bridges to `web-app-matomo`. Today that bridge is implicit. This requirement makes it explicit: **an addon that bridges to another role MUST declare the bridged service, and that service MUST be present in the role's `meta/services.yml`.** @@ -25,23 +25,23 @@ A recurring property of these extensions is that many of them exist **only to br These choices are settled at requirement creation time and bound the implementation. Re-opening any of them MUST be recorded in the implementing PR. -1. **One file, one topic.** Addon definitions live in `roles//meta/addons.yml`. The file-root convention applies: the file content IS the value of `applications..addons`, with no wrapping `addons:` key (see [layout.md](../contributing/design/role/services/layout.md)). The materialised path is `applications..addons.`, read via `lookup('config', application_id, 'addons.')`. -2. **Unified schema, native mechanism preserved.** Every addon entry uses one schema across all roles. The app-native term is retained only as the `mechanism` field (`addon`, `plugin`, `mu_plugin`, `extension`, `module`, `bridge`). Tasks keep using the upstream installer for that mechanism; only the *declaration* is unified. App-specific runtime configuration is carried under an opaque `config:` mapping that only the owning role's tasks interpret, so the surrounding schema stays uniform while each addon keeps its full per-addon payload. `meta/addons.yml` is therefore the single source for both the declaration and the runtime config: where a role previously split the enable/disable declaration from a per-addon config payload (e.g. Nextcloud's [vars/plugins/](../../roles/web-app-nextcloud/vars/plugins/)), both move into `meta/addons.yml` and the superseded per-addon config files are deleted by the migration. +1. **One file per addon.** Addon definitions live under `roles//meta/addons/`, one file per addon at `meta/addons/.yml`. The file-root convention applies per file: the file content IS the addon spec, with no wrapping `:` key (the filename stem supplies the addon id); all files are collected into `applications..addons` keyed by `` (see [layout.md](../contributing/design/role/services/layout.md)). The materialised path is `applications..addons.`, read via `lookup('config', application_id, 'addons.')`. +2. **Unified schema, native mechanism preserved.** Every addon entry uses one schema across all roles. The app-native term is retained only as the `mechanism` field (`addon`, `plugin`, `mu_plugin`, `extension`, `module`, `bridge`). Tasks keep using the upstream installer for that mechanism; only the *declaration* is unified. App-specific runtime configuration is carried under an opaque `config:` mapping that only the owning role's tasks interpret, so the surrounding schema stays uniform while each addon keeps its full per-addon payload. Each `meta/addons/.yml` is therefore the single source for both the declaration and the runtime config: where a role previously split the enable/disable declaration from a per-addon config payload (e.g. Nextcloud's `vars/plugins/`), both move into that addon's `meta/addons/.yml` file and the superseded per-addon config files are deleted by the migration. 3. **Bridges are explicit and validated.** When an addon itself integrates with another role, it MUST list the bridged service key under `bridges:`. Each listed key MUST resolve to a service block already declared in the same role's `meta/services.yml`. Lint fails otherwise. Front-door auth gates such as oauth2-proxy are not addon bridges unless the addon itself talks to that service. **Two distinct meanings of "bridge" exist and MUST NOT be conflated:** the `bridges:` *field* names an in-repo cross-role service dependency (e.g. an addon that talks to `svc-db-openldap`), whereas `mechanism: bridge` marks an addon that *is* a network/appservice bridge to an external system (e.g. a Matrix mautrix bridge to WhatsApp/Telegram). A single addon MAY be both: a `mechanism: bridge` addon MAY also declare a `bridges:` dependency on an in-repo service it relies on. 4. **No new secret store.** Any credential an addon needs continues to be declared in [`meta/schema.yml`](../contributing/design/role/services/layout.md) `credentials:` and read via `lookup('config', application_id, 'credentials.')`. -5. **Enable state defaults to off; deliberately-wired integrations auto-enable with their partner.** `enabled` is optional and defaults to `false` unless `required: true` is set. An addon that bridges a service MUST derive its effective enablement from that service's `enabled` flag (never a duplicated group-membership expression), so a deliberately-wired cross-role integration activates automatically whenever its partner role is co-deployed (`services..enabled` is true) and stays off otherwise. Any credential an addon needs is declared in `meta/schema.yml` `credentials:` and read via `lookup('config', application_id, 'credentials.')` (Decision 4) โ€” auto-generated where possible, or supplied through the same credentials mechanism's manual-entry path where the operator must enter the secret. There is no separate opt-in exception: auto-enable-with-partner holds, and an operator-entered credential resolves through `credentials` like any other. This auto-enable-with-partner rule is distinct from update-automation-discovered addons (Decision 10), which are added hard-off. +5. **Enable state defaults to off; deliberately-wired integrations auto-enable with their partner.** `enabled` is optional and defaults to `false` unless `required: true` is set. An addon that bridges a service MUST derive its effective enablement from that service's `enabled` flag (never a duplicated group-membership expression), so a deliberately-wired cross-role integration activates automatically whenever its partner role is co-deployed (`services..enabled` is true) and stays off otherwise. Any credential an addon needs is declared in `meta/schema.yml` `credentials:` and read via `lookup('config', application_id, 'credentials.')` (Decision 4): auto-generated where possible, or supplied through the same credentials mechanism's manual-entry path where the operator must enter the secret. There is no separate opt-in exception: auto-enable-with-partner holds, and an operator-entered credential resolves through `credentials` like any other. This auto-enable-with-partner rule is distinct from update-automation-discovered addons (Decision 10), which are added hard-off. 6. **Required addons are explicit, and install failure is gated by `required`.** Every addon carries a boolean `required` field (default `false`). Core components that are part of the deployed app contract, such as Odoo's core modules, use `required: true`: they are always installed, MAY omit `enabled`, and MUST NOT set `enabled: false`. The field also governs install-failure behaviour: when an addon's installation fails, a `required: true` addon MUST hard-fail the play (the deploy stops), while a `required: false` addon MUST only emit a warning, skip that addon, and let the play continue. -7. **Workload-neutral contract.** `meta/addons.yml` applies to any role type that declares role-level extensions, including desktop roles such as `desk-gnome-extensions`. The materialised path still uses the role's assembled application entry, `applications..addons`. +7. **Workload-neutral contract.** The `meta/addons/` contract applies to any role type that declares role-level extensions, including desktop roles such as `desk-gnome-extensions`. The materialised path still uses the role's assembled application entry, `applications..addons`. 8. **First slice.** The first end-to-end migration is `web-app-friendica` (`ldapauth`, bridges `ldap` to `svc-db-openldap`), because it already exercises the bridge rule and the service-flag enablement path. 9. **External drift monitoring.** Addon freshness is checked by opt-in external tests under `tests/external/update/addons/`. These tests are warn-only, emit GitHub Actions warning annotations, and are excluded from `make test` in the same way as Docker image and repository-ref external update checks. 10. **Update PR behaviour.** The scheduled update workflow MAY open or refresh an update pull request for addon version bumps and newly discovered catalog entries. Addon entries *discovered by update automation* MUST be added disabled by default (`enabled: false`) and MUST NOT silently enable new runtime behaviour. This hard-off default applies to auto-discovered addons only; it does NOT override the auto-enable-with-partner rule for deliberately-wired integrations (Decision 5). -11. **Every addon ships a Playwright test.** Each addon with a user-facing surface MUST add a matching Playwright spec under `roles//files/playwright/` that exercises that addon's behaviour (e.g. an `ldapauth`/`sociallogin` login path, a `wp-discourse` round-trip, an enabled Odoo module's UI entry point). An addon with no web-facing surface (e.g. a `desk-gnome-extensions` desktop extension) is exempt and MUST carry a one-line note in the role README stating why. No addon is considered implemented or migrated until its Playwright test is present and green, or its exemption is documented. Cross-role integration specs (proving two roles are actually connected) run under a dedicated **integration-pair deploy variant** that co-deploys both partner roles, and are added incrementally as each โœ… pair is wired rather than all upfront. +11. **Every addon ships a Playwright test.** Each addon with a user-facing surface MUST add a matching Playwright spec under `roles//files/playwright/` that exercises that addon's behaviour (e.g. an `ldapauth`/`sociallogin` login path, a `wp-discourse` round-trip, an enabled Odoo module's UI entry point). Desktop roles (`desk-*`) drive no web surface a Playwright test could exercise and therefore NEVER ship addon specs โ€” the lint forbids spec files under `desk-*` roles. For every web-facing role there is no per-addon exemption: an addon is considered migrated only once its Playwright spec is present and green. Cross-role integration specs (proving two roles are actually connected) run under a dedicated **integration-pair deploy variant** that co-deploys both partner roles, and are added incrementally as each โœ… pair is wired rather than all upfront. 12. **Addon state is a variant axis.** Optional addons (`required: false`) MUST express their enabled/disabled split through the role's [`meta/variants.yml`](../contributing/design/role/services/layout.md) so CI matrix runs exercise both states, mirroring requirement 025's MCP variant axis. Required addons (`required: true`) are not a variant axis because they are always installed. -13. **Network bridges are addons.** Appservice / network bridges (e.g. the Matrix `mautrix` bridges to WhatsApp, Telegram, Signal, Slack, Facebook, Instagram) are addons declared in `meta/addons.yml` with `mechanism: bridge`, one entry per bridged network. Each bridge owns its credentials in [`meta/schema.yml`](../contributing/design/role/services/layout.md) and is `required: false` and disabled by default. The role's former bridge declarations (e.g. `web-app-matrix`'s [vars/bridges.yml](../../roles/web-app-matrix/vars/bridges.yml)) are absorbed into `meta/addons.yml`. -14. **The integration matrix is the cross-role addon backlog.** The full roleร—role capability map lives in the generated companion [027-integration-matrix.md](027-integration-matrix.md) (source of truth: [027-integration-matrix.gen.py](027-integration-matrix.gen.py)). โ˜‘๏ธ cells are integrations **already wired** through an infinito-native service flag in the row role's `meta/services.yml` (e.g. `sso`โ†’keycloak, `matomo`, `prometheus`, `email`โ†’mailu) โ€” derived by scanning the repo, no action required. โœ… cells are verified **upstream appโ†”app plugins not yet declared** as addons โ€” this is the backlog this requirement commits to wire. ๐Ÿช™ cells are the same as โœ… but gated behind a commercial tier (opt-in, operator-enabled). โŒ cells have no known integration. The matrix MUST be regenerated (not hand-edited) whenever a role or an integration is added; the `meta/addons.yml` migrations turn โœ… into โ˜‘๏ธ over time. -15. **Scope is the full backlog, executed in phases.** The definition of done is the FULL set: every โœ… cell in the matrix wired as an addon, and an exhaustive per-`web-app-*`-role upstream-plugin survey completed so the โœ… layer is exhaustive rather than hub-biased. Delivery is phased โ€” the shared contract + the first-slice roles land first (see [Implementation Strategy](#implementation-strategy)), then the remaining roles and โœ… wirings sweep incrementally โ€” but no phase reduces scope: the requirement is not complete until the full โœ… set is wired and every `web-app-*` role surveyed. +13. **Network bridges are addons.** Appservice / network bridges (e.g. the Matrix `mautrix` bridges to WhatsApp, Telegram, Signal, Slack, Facebook, Instagram) are addons declared under `meta/addons/` with `mechanism: bridge`, one file per bridged network. Each bridge owns its credentials in [`meta/schema.yml`](../contributing/design/role/services/layout.md) and is `required: false` and disabled by default. The role's former bridge declarations (e.g. `web-app-matrix`'s `vars/bridges.yml`) are absorbed into `meta/addons/`. +14. **The integration matrix is the cross-role addon backlog.** The full roleร—role capability map lives in the generated companion [027-integration-matrix.md](027-integration-matrix.md) (source of truth: [027-integration-matrix.gen.py](027-integration-matrix.gen.py)). โ˜‘๏ธ cells are integrations **already wired** through an infinito-native service flag in the row role's `meta/services.yml` (e.g. `sso`โ†’keycloak, `matomo`, `prometheus`, `email`โ†’mailu), derived by scanning the repo, no action required. โœ… cells are verified **upstream appโ†”app plugins not yet declared** as addons, the backlog this requirement commits to wire. ๐Ÿช™ cells are the same as โœ… but gated behind a commercial tier (opt-in, operator-enabled). โŒ cells have no known integration. The matrix MUST be regenerated (not hand-edited) whenever a role or an integration is added; the `meta/addons/` migrations turn โœ… into โ˜‘๏ธ over time. +15. **Scope is the full backlog, executed in phases.** The definition of done is the FULL set: every โœ… cell in the matrix wired as an addon, and an exhaustive per-`web-app-*`-role upstream-plugin survey completed so the โœ… layer is exhaustive rather than hub-biased. Delivery is phased: the shared contract plus the first-slice roles land first (see [Implementation Strategy](#implementation-strategy)), then the remaining roles and โœ… wirings sweep incrementally, but no phase reduces scope. The requirement is not complete until the full โœ… set is wired and every `web-app-*` role surveyed. 16. **The integration matrix is a generated artifact with a sanctioned generator.** `027-integration-matrix.gen.py` stays under `docs/requirements/` but MUST be refactored to satisfy the repo lints: its `EDGES` data moves into a sibling data module so the entrypoint stays under the 500-line limit, it uses the shared project-root helper instead of a local `__file__` upward walk, and it carries no self-path literal or `requirement NNN` token. The generated `027-integration-matrix.md` carries a `generated-artifact / not-a-requirement` head marker so the requirements-completeness/archival lint skips it. Dead upstream URLs in `EDGES` are fixed or dropped on regeneration. -17. **Only bridging addons add a service block.** A `:` service block is added to the row role ONLY for addons that declare a `bridges:` dependency (per Decision 3) โ€” i.e. the in-repo cross-role integrations; a plain addon with no cross-role dependency adds no service block. Each bridging addon's `:` block carries the standard dynamic `enabled`/`shared` flags with matching true/false `meta/variants.yml` coverage, and โ€” for web-facing pairs โ€” a co-deploy Playwright spec. Across the in-repo โœ… set this is a bounded, mechanical expansion scoped to the bridging integrations, accepted as part of Decision 15's full scope. +17. **Only bridging addons add a service block.** A `:` service block is added to the row role ONLY for addons that declare a `bridges:` dependency (per Decision 3), i.e. the in-repo cross-role integrations; a plain addon with no cross-role dependency adds no service block. Each bridging addon's `:` block carries the standard dynamic `enabled`/`shared` flags with matching true/false `meta/variants.yml` coverage, and (for web-facing pairs) a co-deploy Playwright spec. Across the in-repo โœ… set this is a bounded, mechanical expansion scoped to the bridging integrations, accepted as part of Decision 15's full scope. 18. **Numbering.** This document is `026-unified-addon-syntax.md`, forming the trio `025-mcp-role-integration` โ†’ `026-unified-addon-syntax` โ†’ `027-integration-matrix`. The pre-existing `025-mcp-role-integration` vs `025-web-app-matrix-ansible-flavor` number collision is out of this requirement's scope; a separate doc owns its resolution. ## Current State Audit @@ -52,12 +52,12 @@ Each row is a migration target. The "Bridges" column is the cross-role dependenc |---|---|---|---|---| | [web-app-friendica](../../roles/web-app-friendica/) | `addon` | `ldapauth` | `ldap` โ†’ `svc-db-openldap` | [meta/services.yml](../../roles/web-app-friendica/meta/services.yml) `friendica.addons` | | [web-app-odoo](../../roles/web-app-odoo/) | `module` | `crm, contacts, sale_management, account, website, project, stock` (core); `[]` (optional) | none today | [meta/services.yml](../../roles/web-app-odoo/meta/services.yml) `odoo.modules` | -| [web-app-nextcloud](../../roles/web-app-nextcloud/) | `plugin` | `bbb, onlyoffice, richdocuments, sociallogin, spreed, user_ldap, whiteboard, xwiki` | `sociallogin` โ†’ `web-app-keycloak`, `user_ldap` โ†’ `svc-db-openldap` | [vars/plugins/](../../roles/web-app-nextcloud/vars/plugins/) | -| [web-app-wordpress](../../roles/web-app-wordpress/) | `plugin`, `mu_plugin` | `daggerhart-openid-connect-generic, wp-discourse` (+ mu-plugins) | OIDC โ†’ `web-app-keycloak`, `wp-discourse` โ†’ `web-app-discourse` | [tasks/plugins/](../../roles/web-app-wordpress/tasks/plugins/) | +| [web-app-nextcloud](../../roles/web-app-nextcloud/) | `plugin` | `bbb, onlyoffice, richdocuments, sociallogin, spreed, user_ldap, whiteboard, xwiki` | `sociallogin` โ†’ `web-app-keycloak`, `user_ldap` โ†’ `svc-db-openldap` | `vars/plugins/` | +| [web-app-wordpress](../../roles/web-app-wordpress/) | `plugin`, `mu_plugin` | `daggerhart-openid-connect-generic, wp-discourse` (+ mu-plugins) | OIDC โ†’ `web-app-keycloak`, `wp-discourse` โ†’ `web-app-discourse` | `tasks/plugins/` | | [web-app-mediawiki](../../roles/web-app-mediawiki/) | `extension` | `PluggableAuth, OpenIDConnect` | `OpenIDConnect` โ†’ `web-app-keycloak` | [vars/main.yml](../../roles/web-app-mediawiki/vars/main.yml) | | [web-app-xwiki](../../roles/web-app-xwiki/) | `plugin` | `oidc-authenticator, ldap-authenticator, matomo` | `oidc` โ†’ `web-app-keycloak`, `ldap` โ†’ `svc-db-openldap`, `matomo` โ†’ `web-app-matomo` | [meta/services.yml](../../roles/web-app-xwiki/meta/services.yml) `xwiki.plugins` | | [web-app-joomla](../../roles/web-app-joomla/) | `plugin` | `plg_system_keycloak` | `sso` โ†’ `web-app-keycloak` | [tasks/07_oidc_plugin.yml](../../roles/web-app-joomla/tasks/07_oidc_plugin.yml) | -| [web-app-matrix](../../roles/web-app-matrix/) | `bridge` | `mautrix-whatsapp, mautrix-telegram, mautrix-signal, mautrix-slack, mautrix-facebook, mautrix-instagram` | external networks (no in-repo service); each owns a DB credential | [vars/bridges.yml](../../roles/web-app-matrix/vars/bridges.yml) | +| [web-app-matrix](../../roles/web-app-matrix/) | `bridge` | `mautrix-whatsapp, mautrix-telegram, mautrix-signal, mautrix-slack, mautrix-meta` | external networks (no in-repo service); each owns a DB credential | `vars/bridges.yml` | | [web-app-discourse](../../roles/web-app-discourse/) | `plugin` | `docker_manager, discourse-activity-pub, discourse-akismet, discourse-ldap-auth` | `discourse-ldap-auth` โ†’ `svc-db-openldap` | [meta/services.yml](../../roles/web-app-discourse/meta/services.yml) `discourse.plugins` | | [web-app-pretix](../../roles/web-app-pretix/) | `plugin` | `oidc` (v2.3.1) | `sso` โ†’ `web-app-keycloak` | [meta/services.yml](../../roles/web-app-pretix/meta/services.yml) `pretix.plugins` | | [web-app-mattermost](../../roles/web-app-mattermost/) | `plugin` | volume-managed plugins (`plugins`, `client-plugins`) | none declared today | [vars/main.yml](../../roles/web-app-mattermost/vars/main.yml), [meta/volumes.yml](../../roles/web-app-mattermost/meta/volumes.yml) | @@ -68,48 +68,47 @@ Each row is a migration target. The "Bridges" column is the cross-role dependenc ### Out of scope: database/runtime engine extensions -Two roles declare an `extensions:` key that is **not** an application addon but a database/cache engine feature, provisioned at the storage layer. These MUST stay where they are and MUST NOT move into `meta/addons.yml`: +Two roles declare an `extensions:` key that is **not** an application addon but a database/cache engine feature, provisioned at the storage layer. These MUST stay where they are and MUST NOT move into `meta/addons/`: -- [web-app-mobilizon](../../roles/web-app-mobilizon/) โ€” PostgreSQL extensions `postgis, pg_trgm, unaccent` ([meta/services.yml](../../roles/web-app-mobilizon/meta/services.yml)). -- [web-app-bookwyrm](../../roles/web-app-bookwyrm/) โ€” engine extension `bloom` ([meta/services.yml](../../roles/web-app-bookwyrm/meta/services.yml)). +- [web-app-mobilizon](../../roles/web-app-mobilizon/): PostgreSQL extensions `postgis, pg_trgm, unaccent` ([meta/services.yml](../../roles/web-app-mobilizon/meta/services.yml)). +- [web-app-bookwyrm](../../roles/web-app-bookwyrm/): engine extension `bloom` ([meta/services.yml](../../roles/web-app-bookwyrm/meta/services.yml)). The audit MUST classify these as `db-extension` (out of scope), so a later sweep does not mistake them for app addons. If a future role mixes app addons and DB extensions under one key, the migration MUST split them by destination. -A grep for `meta/addons.yml` before implementation MUST return nothing, and that baseline MUST be recorded in the implementing PR. +A grep for `meta/addons/` before implementation MUST return nothing, and that baseline MUST be recorded in the implementing PR. ## Cross-Role Integration Matrix The companion [027-integration-matrix.md](027-integration-matrix.md) is a generated roleร—role matrix over the entity names of every `web-app-*` and `web-svc-*` role. Each cell marks whether the **row** role wires in the **column** role: -- โ˜‘๏ธ โ€” already wired via an infinito-native service flag in the row role's `meta/services.yml` (cell links to that declaration); -- โœ… โ€” a verified upstream appโ†”app plugin exists but is not yet declared as an addon (cell links to the plugin page) โ€” the backlog; -- ๐Ÿช™ โ€” same as โœ… but gated behind a commercial / paid tier; -- โŒ โ€” no known integration; -- โ€” โ€” diagonal (same role). +- โ˜‘๏ธ : already wired via an infinito-native service flag in the row role's `meta/services.yml` (cell links to that declaration); +- โœ… : a verified upstream appโ†”app plugin exists but is not yet declared as an addon (cell links to the plugin page), the backlog; +- ๐Ÿช™ : same as โœ… but gated behind a commercial / paid tier; +- โŒ : no known integration; +- `โ€”` : diagonal (same role). -The matrix is **directional** (the row hosts the plugin/flag); bidirectional pairs such as `nextcloud`โ†”`openproject` carry a symbol in both cells. It is generated by [027-integration-matrix.gen.py](027-integration-matrix.gen.py): โ˜‘๏ธ edges are scanned from every role's `meta/services.yml` (the `sso`/`matomo`/`prometheus`/`email`/`dashboard`/`css`/`logout`/`cdn`/`coturn`/`collabora`/`onlyoffice`/`libretranslate` keys), and โœ…/๐Ÿช™ edges come from a curated, web-verified `EDGES` map. It MUST be regenerated rather than hand-edited. At creation time it covers 82 roles with **511 infinito-native (โ˜‘๏ธ) wired edges plus 96 curated upstream (โœ…/๐Ÿช™) edges**. The โ˜‘๏ธ layer is exhaustive (it is scanned from the repo); the โœ…/๐Ÿช™ layer is **not yet exhaustive** โ€” it grows as each role's upstream plugin ecosystem is researched in depth (e.g. a MediaWiki deep-dive added `DiscourseSsoConsumer`, `PeerTubeEmbed`, `MachineTranslation`โ†’libretranslate, `Extension:AWS`โ†’minio, and an LLM editorโ†’openwebui). The โœ…/๐Ÿช™ set is the **backlog**: every โœ… pair is an addon this requirement commits to declare and wire (see Acceptance Criteria), and completing the per-role upstream survey is itself an acceptance criterion. +The matrix is **directional** (the row hosts the plugin/flag); bidirectional pairs such as `nextcloud`โ†”`openproject` carry a symbol in both cells. It is generated by [027-integration-matrix.gen.py](027-integration-matrix.gen.py): โ˜‘๏ธ edges are scanned from every role's `meta/services.yml` (the `sso`/`matomo`/`prometheus`/`email`/`dashboard`/`css`/`logout`/`cdn`/`coturn`/`collabora`/`onlyoffice`/`libretranslate` keys), and โœ…/๐Ÿช™ edges come from a curated, web-verified `EDGES` map. It MUST be regenerated rather than hand-edited. At creation time it covers 82 roles with **523 infinito-native (โ˜‘๏ธ) wired edges plus 96 curated upstream (โœ…/๐Ÿช™) edges**. The โ˜‘๏ธ layer is exhaustive (it is scanned from the repo); the โœ…/๐Ÿช™ layer is **not yet exhaustive** and grows as each role's upstream plugin ecosystem is researched in depth (e.g. a MediaWiki deep-dive added `DiscourseSsoConsumer`, `PeerTubeEmbed`, `MachineTranslation`โ†’libretranslate, `Extension:AWS`โ†’minio, and an LLM editorโ†’openwebui). The โœ…/๐Ÿช™ set is the **backlog**: every โœ… pair is an addon this requirement commits to declare and wire (see Acceptance Criteria), and completing the per-role upstream survey is itself an acceptance criterion. ## Target Schema -### Addon declaration: `meta/addons.yml` +### Addon declaration: `meta/addons/.yml` -Every role that ships role-level extensions MUST declare them in `roles//meta/addons.yml`. The file root IS the addons map keyed by ``. There is NO wrapping `addons:` key. +Every role that ships role-level extensions MUST declare them under `roles//meta/addons/`, one file per addon at `meta/addons/.yml`. The file root IS the addon spec; there is NO wrapping `:` key (the filename stem supplies the addon id). ```yaml -# roles/web-app-friendica/meta/addons.yml (file root IS the addons map) -ldapauth: - enabled: "{{ lookup('config', application_id, 'services.ldap.enabled') | bool }}" - required: false # true for core modules that must always install - mechanism: addon # addon | plugin | mu_plugin | extension | module - source: upstream # upstream | bundled | vendored | built - bridges: # optional; service keys declared in meta/services.yml - - ldap - version: "" # optional upstream pin; "" means upstream default - group: optional # optional grouping label (e.g. odoo core/optional) - update: - monitored: true # optional; external tests check latest versions - catalog: friendica-addons # optional; supported upstream catalog adapter - upstream_id: ldapauth # optional; defaults to addon id +# roles/web-app-friendica/meta/addons/ldapauth.yml (file root IS the addon spec) +enabled: "{{ lookup('config', application_id, 'services.ldap.enabled') | bool }}" +required: false # true for core modules that must always install +mechanism: addon # addon | plugin | mu_plugin | extension | module +source: upstream # upstream | bundled | vendored | built +bridges: # optional; service keys declared in meta/services.yml + - ldap +version: "" # optional upstream pin; "" means upstream default +group: optional # optional grouping label (e.g. odoo core/optional) +update: + monitored: true # optional; external tests check latest versions + catalog: friendica-addons # optional; supported upstream catalog adapter + upstream_id: ldapauth # optional; defaults to addon id ``` Field rules: @@ -129,43 +128,41 @@ Field rules: A Nextcloud plugin that previously lived as a standalone config file moves wholesale into its `config:` block: ```yaml -# roles/web-app-nextcloud/meta/addons.yml (excerpt) -sociallogin: - enabled: "{{ lookup('config', application_id, 'services.sso.enabled') | bool }}" - mechanism: plugin - source: upstream - bridges: - - sso - config: - plugin_configuration: - - appid: sociallogin - configkey: custom_providers - configvalue: - custom_oidc: - - name: "{{ lookup('domain','web-app-keycloak') }}" - clientId: "{{ OIDC.CLIENT.ID }}" - clientSecret: "{{ OIDC.CLIENT.SECRET }}" - # ... remaining provider keys, verbatim from the former vars/plugins/sociallogin.yml ... +# roles/web-app-nextcloud/meta/addons/sociallogin.yml (file root IS the addon spec) +enabled: "{{ lookup('config', application_id, 'services.sso.enabled') | bool }}" +mechanism: plugin +source: upstream +bridges: + - sso +config: + plugin_configuration: + - appid: sociallogin + configkey: custom_providers + configvalue: + custom_oidc: + - name: "{{ lookup('domain','web-app-keycloak') }}" + clientId: "{{ OIDC.CLIENT.ID }}" + clientSecret: "{{ OIDC.CLIENT.SECRET }}" + # ... remaining provider keys, verbatim from the former vars/plugins/sociallogin.yml ... ``` -The former [roles/web-app-nextcloud/vars/plugins/sociallogin.yml](../../roles/web-app-nextcloud/vars/plugins/sociallogin.yml) is deleted; the install/enable/configure tasks read `applications.web-app-nextcloud.addons.sociallogin.config` instead. +The former `roles/web-app-nextcloud/vars/plugins/sociallogin.yml` is deleted; the install/enable/configure tasks read `applications.web-app-nextcloud.addons.sociallogin.config` instead. #### Worked example: network bridge addon -A Matrix mautrix bridge becomes one addon entry per bridged network, absorbing the former [vars/bridges.yml](../../roles/web-app-matrix/vars/bridges.yml): +A Matrix mautrix bridge becomes one addon entry per bridged network, absorbing the former `vars/bridges.yml`: ```yaml -# roles/web-app-matrix/meta/addons.yml (excerpt) -mautrix-telegram: - enabled: false - required: false - mechanism: bridge - source: upstream - config: - bridge_name: telegram - database_username: mautrix_telegram_bridge - database_name: mautrix_telegram_bridge - database_password: "{{ lookup('config', application_id, 'credentials.mautrix_telegram_bridge_database_password') }}" +# roles/web-app-matrix/meta/addons/mautrix-telegram.yml (file root IS the addon spec) +enabled: false +required: false +mechanism: bridge +source: upstream +config: + bridge_name: telegram + database_username: mautrix_telegram_bridge + database_name: mautrix_telegram_bridge + database_password: "{{ lookup('config', application_id, 'credentials.mautrix_telegram_bridge_database_password') }}" ``` The bridge's database password stays declared in [meta/schema.yml](../../roles/web-app-matrix/meta/schema.yml) `credentials:` and is referenced, never inlined. Each additional network (`whatsapp`, `signal`, `slack`, `facebook`, `instagram`) is its own `mechanism: bridge` addon. @@ -202,10 +199,28 @@ Rules: ### Materialised path and loader -- `meta/addons.yml..<...>` materialises at `applications..addons..<...>`, consistent with the file-root convention in [layout.md](../contributing/design/role/services/layout.md). -- The existing application loader (`utils/cache/applications.py`) MUST load `meta/addons.yml` like the other file-rooted meta topics. No new generated repository-wide dictionary is introduced. +- `meta/addons/.yml.<...>` materialises at `applications..addons..<...>`, consistent with the file-root convention in [layout.md](../contributing/design/role/services/layout.md). +- The existing application loader (`utils/cache/applications.py`) MUST load every `meta/addons/*.yml` like the other file-rooted meta topics. No new generated repository-wide dictionary is introduced. - The `addons` key MUST be removed from the "inlined under the primary entity" set in [layout.md](../contributing/design/role/services/layout.md) and documented as its own topic file. +### Per-addon task hooks + +- An addon that needs install/enable/configure logic beyond the generic `config:` payload MUST place it in `roles//tasks/addons/.yml`, mirroring the `meta/addons/.yml` file-root convention. +- The role's addon-processing loop MUST include `tasks/addons/.yml` when present and tolerate its absence. +- Existing per-plugin task logic MUST be moved under `tasks/addons/` so the schema is uniform across roles (superseding ad-hoc locations such as the former per-plugin hook directories). + +### Proprietary API credentials (the `API` SPOT) + +- Credentials for proprietary / external (non-self-hosted) providers MUST be declared once in `group_vars/all/18_api.yml` under the `API` mapping (a key per provider, e.g. `cloudflare`, `telegram`, `github`, `google`, `jira`), sourced from the vault and defaulting to empty. +- All access to those values MUST go through the `lookup('api', '.')` plugin ([plugins/lookup/api.py](../../plugins/lookup/api.py)), giving the proprietary-API logic a single point to change. +- An integration whose partner is self-hostable in-repo (e.g. `gitlab`) is wired as a local `bridges:` integration gated on the partner role being present. An integration whose target is purely proprietary / external (e.g. `github`, `google`, `jira`) carries NO `bridges:` and MUST be enabled only when its `lookup('api', '.client_id')` and `client_secret` are present. + +### Per-addon Playwright specs + +- Every addon of a web-facing role MUST ship `roles//files/playwright/addons/.spec.js`. There is NO per-addon exemption: no mechanism-based auto-exempt and no `# nocheck` escape; network bridges and auth extensions covered by the role's OIDC/LDAP login spec all ship their own spec asserting the addon-specific behaviour or skip-on-disabled gating. Desktop (`desk-*`) roles are the one exception โ€” they drive no web surface, so they MUST NOT ship addon specs and the lint forbids them. +- Each addon spec MUST guard on the addon being enabled via `skipUnlessAddonEnabled('')`, backed by the `ADDON__ENABLED` flags rendered into `templates/playwright.env.j2`, and skip cleanly when it is not. +- The `test-e2e-playwright` role MUST stage `files/playwright/` recursively (so `addons/` is included) and discover specs recursively. + ### External update monitoring Addon monitoring MUST follow the existing external-update pattern used for Docker image versions and repository refs. @@ -214,7 +229,7 @@ Rules: - Live upstream checks MUST live under `tests/external/update/addons/` and run through `make test-external`, not through the default `make test` flow. - The external addon test MUST emit warnings, not failures, when a monitored addon has a newer upstream version. -- The external addon test MUST emit warnings, not failures, when a supported catalog adapter detects a new relevant addon that is not listed in `meta/addons.yml`. +- The external addon test MUST emit warnings, not failures, when a supported catalog adapter detects a new relevant addon that is not listed in `meta/addons/`. - Catalog discovery MUST be bounded by explicit adapters and curated relevance rules. The test MUST NOT warn for every package in broad public marketplaces such as the full WordPress plugin directory. - New addon suggestions MUST include enough data for review: role id, addon id, upstream id, mechanism, source, catalog URL, and why the addon is considered relevant. - Version suggestions MUST include the current pinned version, latest upstream version, source file, and line number where possible. @@ -226,50 +241,50 @@ Rules: ### Repository-wide audit - [ ] A deterministic audit command or test enumerates every role under `roles/` and classifies addon support by `mechanism` (`addon`, `plugin`, `mu_plugin`, `extension`, `module`, `bridge`), `db-extension` (out of scope), or `none`. -- [ ] The audit reproduces every row of the [Current State Audit](#current-state-audit) โ€” including the `bridge`-mechanism (`web-app-matrix`), the newly surveyed `plugin`/`extension` roles (`web-app-discourse`, `web-app-pretix`, `web-app-mattermost`, `desk-chromium`, `desk-firefox`, `desk-gnome`), and the out-of-scope `db-extension` roles (`web-app-mobilizon`, `web-app-bookwyrm`) โ€” and records, per role, the addon ids, their `mechanism`, their `required` state, and their `bridges`. -- [ ] A grep for `meta/addons.yml` before implementation is recorded in the implementing PR to show the baseline was empty. +- [ ] The audit reproduces every row of the [Current State Audit](#current-state-audit), including the `bridge`-mechanism (`web-app-matrix`), the newly surveyed `plugin`/`extension` roles (`web-app-discourse`, `web-app-pretix`, `web-app-mattermost`, `desk-chromium`, `desk-firefox`, `desk-gnome`), and the out-of-scope `db-extension` roles (`web-app-mobilizon`, `web-app-bookwyrm`), and records, per role, the addon ids, their `mechanism`, their `required` state, and their `bridges`. +- [ ] A grep for `meta/addons/` before implementation is recorded in the implementing PR to show the baseline was empty. ### Shared contract -- [ ] [layout.md](../contributing/design/role/services/layout.md) documents `meta/addons.yml`: file-root convention, the addon schema, field defaults, allowed values, and the materialised path; and removes `addons` from the inlined-key list. -- [ ] The application loader reads `meta/addons.yml` for every role and exposes `applications..addons.` through `lookup('config', application_id, 'addons.')`. -- [ ] Role-meta lint under [`tests/lint/ansible/services/`](../../tests/lint/ansible/services/) rejects an invalid `mechanism` (outside the `addon`/`plugin`/`mu_plugin`/`extension`/`module`/`bridge` set), an invalid `source`, an invalid `update.catalog`, an empty `bridges: []`, a non-string `version`, a non-boolean `required`, `required: true` combined with `enabled: false`, a non-mapping `config`, a literal secret inlined in `config`, and missing required fields, honouring the `# nocheck:` suppression convention. +- [x] [layout.md](../contributing/design/role/services/layout.md) documents `meta/addons/`: file-root convention, the addon schema, field defaults, allowed values, and the materialised path; and removes `addons` from the inlined-key list. +- [x] The application loader reads `meta/addons/` for every role and exposes `applications..addons.` through `lookup('config', application_id, 'addons.')`. +- [x] Role-meta lint under [`tests/lint/ansible/services/`](../../tests/lint/ansible/services/) rejects an invalid `mechanism` (outside the `addon`/`plugin`/`mu_plugin`/`extension`/`module`/`bridge` set), an invalid `source`, an invalid `update.catalog`, an empty `bridges: []`, a non-string `version`, a non-boolean `required`, `required: true` combined with `enabled: false`, a non-mapping `config`, a literal secret inlined in `config`, and missing required fields, honouring the `# nocheck:` suppression convention. - [ ] After the Nextcloud migration, no `roles/web-app-nextcloud/vars/plugins/*.yml` plugin-config file remains; a test asserts the directory's former per-plugin payloads now live under `applications.web-app-nextcloud.addons..config`. ### Bridge contract -- [ ] Lint fails when an addon's `bridges:` entry names a service key absent from the same role's `meta/services.yml`, and the error message names the role, addon id, and missing service key. -- [ ] Lint verifies that every bridged service block carries `enabled` and `shared` flags. -- [ ] A bridged addon's `enabled` value referencing its single bridged service's `enabled` flag passes lint without a duplicated group-membership expression. -- [ ] Lint allows role-level service dependencies that are not addon bridges, such as Friendica's `sso` front-door auth gate, to remain in `meta/services.yml` without appearing under an unrelated addon's `bridges:`. +- [x] Lint fails when an addon's `bridges:` entry names a service key absent from the same role's `meta/services.yml`, and the error message names the role, addon id, and missing service key. +- [x] Lint verifies that every bridged service block carries `enabled` and `shared` flags. +- [x] A bridged addon's `enabled` value referencing its single bridged service's `enabled` flag passes lint without a duplicated group-membership expression. +- [x] Lint allows role-level service dependencies that are not addon bridges, such as Friendica's `sso` front-door auth gate, to remain in `meta/services.yml` without appearing under an unrelated addon's `bridges:`. ### Per-role migration -- [ ] [web-app-friendica](../../roles/web-app-friendica/) declares `ldapauth` in `meta/addons.yml` with `bridges: [ldap]`; the `ldap` and `sso` service blocks remain in `meta/services.yml`; [tasks/04_addons.yml](../../roles/web-app-friendica/tasks/04_addons.yml) reads the addon list from the new path; deploy behaviour is unchanged. -- [ ] [web-app-odoo](../../roles/web-app-odoo/) declares its `core` and `optional` modules in `meta/addons.yml` using `mechanism: module`, `required: true` for core modules, and the `group` label; the install path reads the new path. -- [ ] [web-app-nextcloud](../../roles/web-app-nextcloud/) declares each plugin in `meta/addons.yml`; `sociallogin` bridges `sso` and `user_ldap` bridges `ldap`, both present in `meta/services.yml`; each plugin's full runtime config from [vars/plugins/](../../roles/web-app-nextcloud/vars/plugins/) is absorbed into its addon `config:` block and the superseded `vars/plugins/*.yml` files are deleted; the install/enable/configure tasks read the addon `config:` payload; deploy behaviour is unchanged. -- [ ] [web-app-wordpress](../../roles/web-app-wordpress/) declares its `plugin` and `mu_plugin` entries in `meta/addons.yml`; the OIDC plugin bridges `sso` and `wp-discourse` bridges the Discourse service. -- [ ] [web-app-mediawiki](../../roles/web-app-mediawiki/) declares `PluggableAuth` and `OpenIDConnect` in `meta/addons.yml` with `mechanism: extension`; `OpenIDConnect` bridges `sso`. -- [ ] [web-app-xwiki](../../roles/web-app-xwiki/) declares its three plugins in `meta/addons.yml`, preserving `version` pins; bridges `sso`, `ldap`, and `matomo` resolve to service blocks. -- [ ] [web-app-joomla](../../roles/web-app-joomla/) declares `plg_system_keycloak` in `meta/addons.yml` with `mechanism: plugin`, `source: built`, and `bridges: [sso]`. -- [ ] [web-app-matrix](../../roles/web-app-matrix/) declares each mautrix network bridge in `meta/addons.yml` with `mechanism: bridge`, `required: false`, `enabled: false` by default, and its DB credential referenced from `meta/schema.yml`; [vars/bridges.yml](../../roles/web-app-matrix/vars/bridges.yml) is absorbed and deleted. -- [ ] [web-app-discourse](../../roles/web-app-discourse/) declares its plugins in `meta/addons.yml` with `mechanism: plugin`; `discourse-ldap-auth` carries `bridges: [ldap]` resolving to the `ldap` service block, while `docker_manager`, `discourse-activity-pub`, and `discourse-akismet` carry no `bridges`. -- [ ] [web-app-pretix](../../roles/web-app-pretix/) declares its `oidc` plugin in `meta/addons.yml` with `mechanism: plugin`, the `2.3.1` pin preserved as a string `version`, and `bridges: [sso]`. -- [ ] [web-app-mattermost](../../roles/web-app-mattermost/) declares its plugins in `meta/addons.yml` with `mechanism: plugin`; the plugin volumes remain in `meta/volumes.yml`. -- [ ] [desk-chromium](../../roles/desk-chromium/) and [desk-firefox](../../roles/desk-firefox/) declare their browser extensions in `meta/addons.yml` with `mechanism: extension`, no `bridges`, and a documented Playwright exemption (no in-app web surface to drive). -- [ ] [desk-gnome](../../roles/desk-gnome/) declares its gnome-shell extensions in `meta/addons.yml` with `mechanism: extension` and no `bridges`. -- [ ] [desk-gnome-extensions](../../roles/desk-gnome-extensions/) declares its extensions in `meta/addons.yml` with `mechanism: extension` and no `bridges`. -- [ ] [web-app-mobilizon](../../roles/web-app-mobilizon/) and [web-app-bookwyrm](../../roles/web-app-bookwyrm/) keep their `extensions:` (PostgreSQL/engine extensions) in `meta/services.yml`; they are classified `db-extension` and explicitly NOT migrated to `meta/addons.yml`. +- [ ] [web-app-friendica](../../roles/web-app-friendica/) declares `ldapauth` in `meta/addons/` with `bridges: [ldap]`; the `ldap` and `sso` service blocks remain in `meta/services.yml`; [tasks/04_addons.yml](../../roles/web-app-friendica/tasks/04_addons.yml) reads the addon list from the new path; deploy behaviour is unchanged. +- [ ] [web-app-odoo](../../roles/web-app-odoo/) declares its `core` and `optional` modules in `meta/addons/` using `mechanism: module`, `required: true` for core modules, and the `group` label; the install path reads the new path. +- [x] [web-app-nextcloud](../../roles/web-app-nextcloud/) declares each plugin in `meta/addons/`; `sociallogin` bridges `sso` and `user_ldap` bridges `ldap`, both present in `meta/services.yml`; each plugin's full runtime config from `vars/plugins/` is absorbed into its addon `config:` block and the superseded `vars/plugins/*.yml` files are deleted; the install/enable/configure tasks read the addon `config:` payload; deploy behaviour is unchanged. +- [ ] [web-app-wordpress](../../roles/web-app-wordpress/) declares its `plugin` and `mu_plugin` entries in `meta/addons/`; the OIDC plugin bridges `sso` and `wp-discourse` bridges the Discourse service. +- [ ] [web-app-mediawiki](../../roles/web-app-mediawiki/) declares `PluggableAuth` and `OpenIDConnect` in `meta/addons/` with `mechanism: extension`; `OpenIDConnect` bridges `sso`. +- [ ] [web-app-xwiki](../../roles/web-app-xwiki/) declares its three plugins in `meta/addons/`, preserving `version` pins; bridges `sso`, `ldap`, and `matomo` resolve to service blocks. +- [ ] [web-app-joomla](../../roles/web-app-joomla/) declares `plg_system_keycloak` in `meta/addons/` with `mechanism: plugin`, `source: built`, and `bridges: [sso]`. +- [ ] [web-app-matrix](../../roles/web-app-matrix/) declares each mautrix network bridge in `meta/addons/` with `mechanism: bridge`, `required: false`, `enabled: false` by default, and its DB credential referenced from `meta/schema.yml`; `vars/bridges.yml` is absorbed and deleted. +- [ ] [web-app-discourse](../../roles/web-app-discourse/) declares its plugins in `meta/addons/` with `mechanism: plugin`; `discourse-ldap-auth` carries `bridges: [ldap]` resolving to the `ldap` service block, while `docker_manager`, `discourse-activity-pub`, and `discourse-akismet` carry no `bridges`. +- [ ] [web-app-pretix](../../roles/web-app-pretix/) declares its `oidc` plugin in `meta/addons/` with `mechanism: plugin`, the `2.3.1` pin preserved as a string `version`, and `bridges: [sso]`. +- [ ] [web-app-mattermost](../../roles/web-app-mattermost/) declares its plugins in `meta/addons/` with `mechanism: plugin`; the plugin volumes remain in `meta/volumes.yml`. +- [x] [desk-chromium](../../roles/desk-chromium/) and [desk-firefox](../../roles/desk-firefox/) declare their browser extensions in `meta/addons/` with `mechanism: extension`, no `bridges`, and ship NO Playwright specs (desktop roles have no web surface; the lint forbids specs under `desk-*`). +- [x] [desk-gnome](../../roles/desk-gnome/) declares its gnome-shell extensions in `meta/addons/` with `mechanism: extension` and no `bridges`. +- [ ] [desk-gnome-extensions](../../roles/desk-gnome-extensions/) declares its extensions in `meta/addons/` with `mechanism: extension` and no `bridges`. +- [x] [web-app-mobilizon](../../roles/web-app-mobilizon/) and [web-app-bookwyrm](../../roles/web-app-bookwyrm/) keep their `extensions:` (PostgreSQL/engine extensions) in `meta/services.yml`; they are classified `db-extension` and explicitly NOT migrated to `meta/addons/`. ### Cross-role integration (matrix backlog) - [ ] [027-integration-matrix.md](027-integration-matrix.md) is regenerated from [027-integration-matrix.gen.py](027-integration-matrix.gen.py) and committed; a check fails if the committed matrix differs from a fresh generation (no hand edits). The generator derives โ˜‘๏ธ edges by scanning every role's `meta/services.yml` integration keys and renders โœ…/๐Ÿช™ from the curated `EDGES` map. - [ ] The matrix axes contain the entity name of **every** `web-app-*` and `web-svc-*` role; a role added later forces a matrix regeneration. -- [ ] โ˜‘๏ธ cells (infinito-native service flags already declared in `meta/services.yml`) are NOT re-declared as `meta/addons.yml` addons โ€” they remain service integrations; only โœ…/๐Ÿช™ cells become new addons. +- [ ] โ˜‘๏ธ cells (infinito-native service flags already declared in `meta/services.yml`) are NOT re-declared as `meta/addons/` addons; they remain service integrations, and only โœ…/๐Ÿช™ cells become new addons. - [ ] A per-role upstream-plugin survey is completed for **every** `web-app-*` role, not only the integration hubs, so the โœ…/๐Ÿช™ layer is exhaustive rather than hub-biased. Each survey records, per partner role, the real upstream plugin (or its documented absence) with a source URL; the MediaWiki survey (which surfaced `DiscourseSsoConsumer`, `PeerTubeEmbed`, `MachineTranslation`โ†’libretranslate, `Extension:AWS`โ†’minio, an LLM editorโ†’openwebui) is the template for depth. -- [ ] **Every โœ… cell is wired**: for each โœ… pair (row โ†’ column), the row role declares an addon in its `meta/addons.yml` that integrates the column role, with `mechanism` set appropriately and โ€” when the column role is an in-repo role โ€” a `bridges:` entry resolving to the row role's `meta/services.yml` service for that target. +- [ ] **Every โœ… cell is wired**: for each โœ… pair (row โ†’ column), the row role declares an addon in its `meta/addons/` that integrates the column role, with `mechanism` set appropriately and (when the column role is an in-repo role) a `bridges:` entry resolving to the row role's `meta/services.yml` service for that target. - [ ] Each wired integration addon is `required: false` and disabled by default, and its enablement is gated on the target role being present (the relevant `services..enabled` flag), so an integration never activates when its partner role is not deployed. -- [ ] **๐Ÿช™ (commercial) cells are opt-in**: they are declared in `meta/addons.yml` but remain disabled, and the role README documents the required commercial tier; they are NOT enabled by default and do not block the requirement. +- [ ] **๐Ÿช™ (commercial) cells are opt-in**: they are declared in `meta/addons/` but remain disabled, and the role README documents the required commercial tier; they are NOT enabled by default and do not block the requirement. - [ ] Where the cell links to an already-wired plugin (e.g. Nextcloudโ†”Collabora/BBB, WordPressโ†”Discourse), the migration reconciles the existing wiring with the matrix entry instead of duplicating it. - [ ] Each newly wired โœ… integration with a user-facing surface ships a Playwright spec (per Confirmed Decision 11) proving the two roles are actually connected when both are deployed. - [ ] SSO-to-`keycloak` cells already satisfied centrally by the `sso` service (requirement 021) are marked as such and are NOT re-implemented as duplicate per-role OIDC addons. @@ -277,29 +292,39 @@ Rules: ### External drift and update PRs - [ ] `tests/external/update/addons/` contains a warn-only external test that checks every addon with `update.monitored=true` for newer upstream versions. -- [ ] The external addon test emits GitHub Actions warning annotations for outdated addon versions and for relevant new catalog entries that are not listed in `meta/addons.yml`. +- [ ] The external addon test emits GitHub Actions warning annotations for outdated addon versions and for relevant new catalog entries that are not listed in `meta/addons/`. - [ ] The external addon test is runnable through `make test-external` and is intentionally excluded from the default `make test` flow. -- [ ] The supported catalog adapters are documented and bounded so broad marketplaces do not produce unreviewable warning noise. +- [x] The supported catalog adapters are documented and bounded so broad marketplaces do not produce unreviewable warning noise. - [ ] The scheduled update workflow opens or refreshes an update pull request when addon versions can be bumped or when relevant new addon entries can be added. - [ ] Update automation adds newly discovered addons with `enabled: false`, `required: false`, complete `update` metadata, and no credentials. - [ ] Update automation preserves existing enablement, bridge, required, and credential-related fields when bumping addon versions. ### Tests -- [ ] Unit or integration tests validate the addon schema and reject unsafe or malformed entries. -- [ ] A parity test (in the spirit of requirement 019) asserts that every addon `bridges:` key has a matching `meta/services.yml` block, repo-wide. -- [ ] A schema test verifies that `enabled` may be omitted, defaults to `false` for optional addons, defaults to true for `required: true`, and rejects `required: true` with `enabled: false`. +- [x] Unit or integration tests validate the addon schema and reject unsafe or malformed entries. +- [x] A parity test (in the spirit of requirement 019) asserts that every addon `bridges:` key has a matching `meta/services.yml` block, repo-wide. +- [x] A schema test verifies that `enabled` may be omitted, defaults to `false` for optional addons, defaults to true for `required: true`, and rejects `required: true` with `enabled: false`. - [ ] An install-failure test proves that a failing `required: true` addon hard-fails the play, while a failing `required: false` addon emits a warning, is skipped, and the play continues. - [ ] Each role with optional (`required: false`) addons expresses the enabled/disabled split as a `meta/variants.yml` axis so the CI matrix exercises both states (mirroring requirement 025). -- [ ] Unit tests cover the external addon discovery logic with fixture catalogs for a version bump, a newly discovered addon, and a marketplace entry intentionally ignored by the curated relevance rules. +- [x] Unit tests cover the external addon discovery logic with fixture catalogs for a version bump, a newly discovered addon, and a marketplace entry intentionally ignored by the curated relevance rules. - [ ] Each migrated role's existing deploy path stays green; no addon behaviour regresses. - [ ] LDAP- and SSO-bridging addons remain covered by the existing Playwright LDAP/SSO requirements (017/018) without new per-role wiring. -- [ ] Every addon with a user-facing surface ships a matching Playwright spec under `roles//files/playwright/` covering that addon's behaviour, and that spec is green before the addon is marked migrated; web-surface-less addons carry a documented README exemption instead. +- [ ] Every addon of a web-facing role ships a matching Playwright spec under `roles//files/playwright/addons/` covering that addon's behaviour, and that spec is green before the addon is marked migrated. Desktop (`desk-*`) roles ship NO specs (the lint forbids them). + +### Per-addon tasks, specs, and the API SPOT + +- [x] A lint (`tests/lint/ansible/services/test_addons_playwright_spec.py`) requires, for every addon of a web-facing role, a `roles//files/playwright/addons/.spec.js` (no mechanism-based auto-exempt and no `# nocheck` escape) and forbids specs under desktop `desk-*` roles; it passes repo-wide. +- [x] The `test-e2e-playwright` role stages `files/playwright/` recursively and discovers specs recursively, so `files/playwright/addons/*.spec.js` are staged and run. +- [x] Each addon spec guards on the addon being enabled (`skipUnlessAddonEnabled`) via the `ADDON__ENABLED` flags rendered into `templates/playwright.env.j2`. +- [x] Per-addon task hooks live in `roles//tasks/addons/.yml`; the former per-plugin task logic was moved there for a uniform schema. +- [x] Proprietary / external API credentials are declared once in `group_vars/all/18_api.yml` (`API` mapping) and accessed exclusively through the `lookup('api', '.')` plugin; the existing cloudflare/telegram code routes through it. +- [x] Self-hostable integration partners (e.g. `gitlab`) are wired as local `bridges:` integrations; purely proprietary targets (`github`, `google`, `jira`) carry no `bridges:` and are enabled only when their `API` credentials are present. +- [ ] Each cross-role integration spec drives the full OAuth connect-flow and is green when both the host and partner roles are deployed (deploy-gated; skips cleanly when the integration is not provisioned). ### Documentation -- [ ] [layout.md](../contributing/design/role/services/layout.md) and the [plugins README](../../plugins/README.md) cross-reference each other so the Ansible-`plugins/` concept and the application-`addons` concept are not conflated. -- [ ] Each migrated role README documents its addons, their `mechanism`, default state, and which services they bridge. +- [x] [layout.md](../contributing/design/role/services/layout.md) and the [plugins README](../../plugins/README.md) cross-reference each other so the Ansible-`plugins/` concept and the application-`addons` concept are not conflated. +- [x] Each migrated role README documents its addons, their `mechanism`, default state, and which services they bridge. - [ ] This requirement file is cross-linked from the implementing PR. ## Validation Apps @@ -319,7 +344,7 @@ Before starting implementation work, the agent MUST read [AGENTS.md](../../AGENT ## Implementation Strategy -1. Add the `meta/addons.yml` schema and loader support, the `tests/lint/ansible/services/` lint rules (schema + bridge resolution), and the [layout.md](../contributing/design/role/services/layout.md) documentation. +1. Add the `meta/addons/` schema and loader support, the `tests/lint/ansible/services/` lint rules (schema + bridge resolution), and the [layout.md](../contributing/design/role/services/layout.md) documentation. 2. Add the repo-wide bridge-parity test. 3. Add the external addon update checker, its fixture-backed unit tests, and update-workflow integration for warn-only drift detection and update pull requests. 4. Migrate `web-app-friendica` as the first end-to-end slice (addon list + bridge resolution + task rewire) and prove deploy parity. diff --git a/docs/requirements/integration_matrix_data.py b/docs/requirements/integration_matrix_data.py new file mode 100644 index 0000000000..ce33dd6e5b --- /dev/null +++ b/docs/requirements/integration_matrix_data.py @@ -0,0 +1,549 @@ +# nocheck: file-size: curated edge data table; one obvious home for the +# matrix's source-of-truth lists, splitting it only adds indirection. +"""Data tables for the roleร—role integration matrix generator. + +Kept as a sibling importable module so the generator entrypoint stays small +and the curated edge data has one obvious home. The generator reads: + +* ``WEB_APP`` / ``WEB_SVC``: the entity-name axes (role suffix of every + ``web-app-*`` / ``web-svc-*`` role); +* ``FRAMEWORK``: top-level ``meta/services.yml`` integration keys mapped to + the in-axis target entity they wire the role into (the infinito-native, + group-gated service flags), distinct from the curated upstream edges; +* ``EDGES``: the curated, web-verified upstream appโ†”app plugin/integration + pages, each ``(source, target, url, kind)`` with ``kind`` in + ``{"check", "coin"}``. +""" + +from __future__ import annotations + +# Top-level meta/services.yml integration keys -> the in-axis target entity +# they wire the role into. Skipped on purpose: +# ldap/redis/mariadb/database -> svc-db-* (not on the axes). +FRAMEWORK: dict[str, str] = { + "sso": "keycloak", + "matomo": "matomo", + "prometheus": "prometheus", + "email": "mailu", + "dashboard": "dashboard", + "css": "css", + "logout": "logout", + "cdn": "cdn", + "coturn": "coturn", + "collabora": "collabora", + "onlyoffice": "onlyoffice", + "libretranslate": "libretranslate", +} + +# Entity names = role suffix of every web-app-* role. +WEB_APP: list[str] = [ + "akaunting", + "baserow", + "bigbluebutton", + "bluesky", + "bookwyrm", + "bridgy-fed", + "chess", + "confluence", + "dashboard", + "decidim", + "discourse", + "erpnext", + "espocrm", + "fediwall", + "fider", + "flowise", + "friendica", + "funkwhale", + "fusiondirectory", + "gitea", + "gitlab", + "hugo", + "jenkins", + "jira", + "jitsi", + "joomla", + "keycloak", + "kix", + "lam", + "listmonk", + "littlejs", + "magento", + "mailu", + "mastodon", + "matomo", + "matrix", + "mattermost", + "mediawiki", + "mig", + "minio", + "mini-qr", + "mobilizon", + "moodle", + "navigator", + "nextcloud", + "odoo", + "opencloud", + "openproject", + "opentalk", + "openwebui", + "peertube", + "pgadmin", + "phpldapadmin", + "phpmyadmin", + "pixelfed", + "postmarks", + "pretix", + "prometheus", + "roulette-wheel", + "shopware", + "snipe-it", + "socialhome", + "sphinx", + "suitecrm", + "taiga", + "wordpress", + "xwiki", + "yourls", + "zammad", +] +WEB_SVC: list[str] = [ + "asset", + "cdn", + "collabora", + "coturn", + "css", + "file", + "html", + "legal", + "libretranslate", + "logout", + "onlyoffice", + "simpleicons", + "xmpp", +] +ENTITIES: list[str] = WEB_APP + WEB_SVC + +# (source, target, url, kind). kind: "check" or "coin". +# Verified upstream plugin/integration pages. +EDGES: list[tuple[str, str, str, str]] = [ + # --- Nextcloud integration_* family + office/auth/analytics --- + ( + "nextcloud", + "openproject", + "https://github.com/nextcloud/integration_openproject", + "check", + ), + ("nextcloud", "gitlab", "https://github.com/nextcloud/integration_gitlab", "check"), + ( + "nextcloud", + "discourse", + "https://github.com/nextcloud/integration_discourse", + "check", + ), + ( + "nextcloud", + "mattermost", + "https://github.com/nextcloud/integration_mattermost", + "check", + ), + ("nextcloud", "matrix", "https://github.com/nextcloud/integration_matrix", "check"), + ("nextcloud", "jira", "https://github.com/nextcloud/integration_jira", "check"), + ("nextcloud", "zammad", "https://github.com/nextcloud/integration_zammad", "check"), + ( + "nextcloud", + "openwebui", + "https://github.com/nextcloud/integration_openai", + "check", + ), + ( + "nextcloud", + "flowise", + "https://github.com/nextcloud/integration_openai", + "check", + ), + ( + "nextcloud", + "mastodon", + "https://github.com/nextcloud/integration_mastodon", + "check", + ), + ( + "nextcloud", + "peertube", + "https://github.com/nextcloud/integration_peertube", + "check", + ), + ("nextcloud", "moodle", "https://github.com/nextcloud/integration_moodle", "check"), + ( + "nextcloud", + "suitecrm", + "https://github.com/eneiluj/integration_suitecrm", + "check", + ), + ("nextcloud", "bigbluebutton", "https://apps.nextcloud.com/apps/bbb", "check"), + ("nextcloud", "xwiki", "https://apps.nextcloud.com/apps/xwiki", "check"), + ("nextcloud", "collabora", "https://github.com/nextcloud/richdocuments", "check"), + ( + "nextcloud", + "onlyoffice", + "https://github.com/ONLYOFFICE/onlyoffice-nextcloud", + "check", + ), + ("nextcloud", "keycloak", "https://github.com/nextcloud/user_oidc", "check"), + ("nextcloud", "matomo", "https://apps.nextcloud.com/apps/matomo", "check"), + # --- Mattermost plugins --- + ( + "mattermost", + "gitlab", + "https://github.com/mattermost/mattermost-plugin-gitlab", + "check", + ), + ( + "mattermost", + "jira", + "https://github.com/mattermost/mattermost-plugin-jira", + "check", + ), + ( + "mattermost", + "jenkins", + "https://github.com/mattermost/mattermost-plugin-jenkins", + "check", + ), + ( + "mattermost", + "jitsi", + "https://github.com/mattermost-community/mattermost-plugin-jitsi", + "check", + ), + ( + "mattermost", + "openwebui", + "https://github.com/mattermost/mattermost-plugin-agents", + "check", + ), + ( + "mattermost", + "prometheus", + "https://github.com/cpanato/mattermost-plugin-alertmanager", + "check", + ), + ( + "mattermost", + "keycloak", + "https://docs.mattermost.com/onboard/sso-saml.html", + "coin", + ), + # --- GitLab native integrations --- + ( + "gitlab", + "mattermost", + "https://docs.gitlab.com/user/project/integrations/mattermost_notifications/", + "check", + ), + ("gitlab", "jira", "https://docs.gitlab.com/integration/jira/", "check"), + ("gitlab", "jenkins", "https://docs.gitlab.com/integration/jenkins/", "check"), + ( + "gitlab", + "matrix", + "https://docs.gitlab.com/user/project/integrations/matrix/", + "check", + ), + ( + "gitlab", + "confluence", + "https://docs.gitlab.com/user/project/integrations/confluence/", + "check", + ), + ("gitlab", "keycloak", "https://docs.gitlab.com/integration/saml/", "check"), + # --- Discourse plugins --- + ( + "discourse", + "mattermost", + "https://github.com/discourse/discourse-chat-integration", + "check", + ), + ( + "discourse", + "matrix", + "https://github.com/discourse/discourse-chat-integration", + "check", + ), + ( + "discourse", + "prometheus", + "https://github.com/discourse/discourse-prometheus", + "check", + ), + ( + "discourse", + "bigbluebutton", + "https://github.com/discourse/discourse-bbb", + "check", + ), + ("discourse", "jitsi", "https://github.com/discourse/discourse-jitsi", "check"), + ("discourse", "openwebui", "https://github.com/discourse/discourse-ai", "check"), + ( + "discourse", + "matomo", + "https://github.com/discourse/discourse-matomo-analytics", + "check", + ), + ( + "discourse", + "keycloak", + "https://github.com/discourse/discourse-openid-connect", + "check", + ), + ( + "discourse", + "mastodon", + "https://github.com/discourse/discourse-activity-pub", + "check", + ), + # --- Matrix (Synapse / hookshot / Element) --- + ("matrix", "gitlab", "https://github.com/matrix-org/matrix-hookshot", "check"), + ("matrix", "jira", "https://github.com/matrix-org/matrix-hookshot", "check"), + ( + "matrix", + "keycloak", + "https://matrix-org.github.io/synapse/latest/openid.html", + "check", + ), + ( + "matrix", + "prometheus", + "https://matrix-org.github.io/synapse/latest/metrics-howto.html", + "check", + ), + ( + "matrix", + "jitsi", + "https://github.com/element-hq/element-web/blob/develop/docs/jitsi.md", + "check", + ), + # --- Moodle plugins --- + ( + "moodle", + "bigbluebutton", + "https://moodle.org/plugins/mod_bigbluebuttonbn", + "check", + ), + ("moodle", "nextcloud", "https://moodle.org/plugins/repository_owncloud", "check"), + ("moodle", "matrix", "https://docs.moodle.org/en/Matrix", "check"), + ("moodle", "keycloak", "https://moodle.org/plugins/auth_oidc", "check"), + ("moodle", "jitsi", "https://moodle.org/plugins/mod_jitsi", "check"), + ("moodle", "matomo", "https://moodle.org/plugins/tool_webanalytics", "check"), + ("moodle", "openwebui", "https://moodle.org/plugins/aiprovider_openwebui", "check"), + # --- Jenkins plugins --- + ("jenkins", "gitlab", "https://plugins.jenkins.io/gitlab-plugin", "check"), + ("jenkins", "gitea", "https://plugins.jenkins.io/gitea/", "check"), + ("jenkins", "mattermost", "https://plugins.jenkins.io/mattermost/", "check"), + ("jenkins", "prometheus", "https://plugins.jenkins.io/prometheus", "check"), + ("jenkins", "keycloak", "https://plugins.jenkins.io/oic-auth/", "check"), + # --- OpenProject --- + ( + "openproject", + "nextcloud", + "https://www.openproject.org/docs/system-admin-guide/integrations/nextcloud/", + "check", + ), + ( + "openproject", + "gitlab", + "https://www.openproject.org/docs/system-admin-guide/integrations/gitlab-integration/", + "check", + ), + ( + "openproject", + "keycloak", + "https://www.openproject.org/docs/system-admin-guide/authentication/openid-providers/", + "coin", + ), + # --- WordPress plugins --- + ("wordpress", "matomo", "https://wordpress.org/plugins/matomo/", "check"), + ("wordpress", "mastodon", "https://wordpress.org/plugins/activitypub/", "check"), + ("wordpress", "discourse", "https://wordpress.org/plugins/wp-discourse/", "check"), + ( + "wordpress", + "keycloak", + "https://wordpress.org/plugins/daggerhart-openid-connect-generic/", + "check", + ), + ( + "wordpress", + "listmonk", + "https://wordpress.org/plugins/integration-for-listmonk-mailing-list-and-newsletter-manager/", + "check", + ), + ( + "wordpress", + "bigbluebutton", + "https://wordpress.org/plugins/video-conferencing-with-bbb/", + "check", + ), + ( + "wordpress", + "peertube", + "https://wordpress.org/plugins/video-manager-for-peertube/", + "check", + ), + # --- MediaWiki extensions --- + ( + "mediawiki", + "keycloak", + "https://www.mediawiki.org/wiki/Extension:OpenID_Connect", + "check", + ), + ("mediawiki", "matomo", "https://www.mediawiki.org/wiki/Extension:Matomo", "check"), + ( + "mediawiki", + "discourse", + "https://www.mediawiki.org/wiki/Extension:DiscourseSsoConsumer", + "check", + ), + ( + "mediawiki", + "peertube", + "https://www.mediawiki.org/wiki/Extension:PeerTubeEmbed", + "check", + ), + ( + "mediawiki", + "libretranslate", + "https://www.mediawiki.org/wiki/Extension:MachineTranslation", + "check", + ), + ("mediawiki", "minio", "https://www.mediawiki.org/wiki/Extension:AWS", "check"), + ( + "mediawiki", + "openwebui", + "https://www.mediawiki.org/wiki/Extension:AIEditingAssistant", + "check", + ), + # --- XWiki extensions --- + ( + "xwiki", + "keycloak", + "https://extensions.xwiki.org/xwiki/bin/view/Extension/OpenID%20Connect/", + "check", + ), + ( + "xwiki", + "matomo", + "https://extensions.xwiki.org/xwiki/bin/view/Extension/Piwiki%20Integration", + "check", + ), + # --- Joomla --- + ( + "joomla", + "keycloak", + "../../roles/web-app-joomla/files/joomla-oidc-plugin/", + "check", + ), + ( + "joomla", + "matomo", + "https://extensions.joomla.org/extension/itcs-matomo/", + "check", + ), + # --- Friendica --- + ("friendica", "matomo", "https://github.com/friendica/friendica-addons", "check"), + # --- Pretix --- + ( + "pretix", + "keycloak", + "https://docs.pretix.eu/en/latest/admin/installation/index.html", + "check", + ), + # --- PeerTube --- + ( + "peertube", + "keycloak", + "https://www.npmjs.com/package/peertube-plugin-auth-openid-connect", + "check", + ), + # --- BigBlueButton (Greenlight) --- + ( + "bigbluebutton", + "keycloak", + "https://docs.bigbluebutton.org/greenlight/v3/external-authentication/", + "check", + ), + # --- CRM / ERP: auth + email --- + ( + "zammad", + "keycloak", + "https://admin-docs.zammad.org/en/latest/settings/security/third-party/saml.html", + "check", + ), + ( + "zammad", + "mailu", + "https://admin-docs.zammad.org/en/latest/channels/email/index.html", + "check", + ), + ("espocrm", "keycloak", "https://docs.espocrm.com/administration/oidc/", "check"), + ( + "espocrm", + "mailu", + "https://docs.espocrm.com/user-guide/imap-smtp-configuration/", + "check", + ), + ( + "suitecrm", + "keycloak", + "https://docs.suitecrm.com/8.x/admin/configuration/saml/8.7.0-saml-configuration/", + "check", + ), + ( + "suitecrm", + "mailu", + "https://github.com/SuiteCRM/SuiteCRM", + "check", + ), + ( + "odoo", + "keycloak", + "https://github.com/OCA/server-auth/tree/16.0/auth_oidc", + "check", + ), + ( + "odoo", + "mailu", + "https://www.odoo.com/documentation/latest/applications/general/email_communication.html", + "check", + ), + ( + "odoo", + "nextcloud", + "https://apps.odoo.com/apps/modules/19.0/nextcloud_odoo_integration", + "coin", + ), + ( + "erpnext", + "keycloak", + "https://docs.frappe.io/framework/user/en/guides/integration/openid_connect_and_frappe_social_login", + "check", + ), + ( + "erpnext", + "mailu", + "https://docs.frappe.io/erpnext/user/manual/en/setting-up-email-account", + "check", + ), + # --- Other SSO-to-Keycloak (covered centrally today via the sso service) --- + ( + "baserow", + "keycloak", + "https://baserow.io/user-docs/configure-openid-connect-for-oauth-2-sso", + "coin", + ), + ( + "taiga", + "keycloak", + "https://github.com/taigaio/taiga-contrib-oidc-auth", + "check", + ), +] diff --git a/group_vars/all/18_api.yml b/group_vars/all/18_api.yml new file mode 100644 index 0000000000..7b05738278 --- /dev/null +++ b/group_vars/all/18_api.yml @@ -0,0 +1,23 @@ +--- +# Central SPOT for proprietary / external (non-self-hosted) API credentials. +# Each entry is empty by default; an integration that depends on a provider is +# only activated when its credentials are present (see the addons that gate +# their `enabled` on `API.`). Override per provider in the (vault- +# encrypted) inventory; access ONLY via lookup('api', '.'). +# nocheck: unused-var: consumed by the api lookup plugin (plugins/lookup/api.py) via lookup('api', ...), not referenced directly in YAML/Jinja. +API: + cloudflare: + api_token: "" + telegram: + bot_token: "" + api_id: "" + api_hash: "" + github: + client_id: "" + client_secret: "" + google: + client_id: "" + client_secret: "" + jira: + client_id: "" + client_secret: "" diff --git a/inventories/development/default.yml b/inventories/development/default.yml index 219be7f406..7023d5be59 100644 --- a/inventories/development/default.yml +++ b/inventories/development/default.yml @@ -22,7 +22,7 @@ BUILDKIT_ENABLED: "{{ lookup('env', 'INFINITO_BUILDKIT_ENABLED') | bool }}" INFINITO_IMAGE_TAG: "{{ lookup('env', 'INFINITO_IMAGE_TAG') | default('latest', true) }}" INFINITO_GHCR_MIRROR_PREFIX: "{{ lookup('env', 'INFINITO_GHCR_MIRROR_PREFIX') | default('', true) }}" CA_CONTAINER_ENABLED: "{{ lookup('env', 'INFINITO_CONTAINER_CA_ENABLED') | bool }}" -TEST_E2E_PLAYWRIGHT_KEEP: "{{ lookup('env', 'INFINITO_PLAYWRIGHT_KEEP') | bool }}" +test_e2e_playwright_keep: "{{ lookup('env', 'INFINITO_PLAYWRIGHT_KEEP') | bool }}" networks: internet: ip4: "{{ lookup('env', 'INFINITO_IP4') }}" diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD index be831efa6c..85bea14468 100644 --- a/packaging/arch/PKGBUILD +++ b/packaging/arch/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: Kevin Veen-Birkenbach pkgname=infinito-nexus -pkgver=9.3.0 +pkgver=11.1.0 pkgrel=1 pkgdesc="Meta package for Infinito.Nexus host dependencies (dev + CI)." arch=('any') diff --git a/packaging/debian/changelog b/packaging/debian/changelog index 6d7d562189..f1bb9858e6 100644 --- a/packaging/debian/changelog +++ b/packaging/debian/changelog @@ -1,36 +1,135 @@ +infinito-nexus (11.1.0-1) unstable; urgency=medium + + * Trusted fork PRs: a maintainer-applied *trusted-pr* label runs a fork PR as deliberately released code โ€” the orchestrator skips the fork-prerequisite wait and the privileged *pull_request_target* build switches its source from the base merge ref to the raw PR head, while the built image keeps the merge-SHA tag so every downstream consumer is unchanged. The label is the only switch (applying a label is collaborator-only on GitHub, so fork authors cannot grant it), build-orchestration scripts always come from the base repository, and both unlabeled fork PRs and same-repository PRs are untouched. See [the fork-PR pipeline](docs/contributing/artefact/git/pipeline.md) and [the contributor fork workflow](docs/contributing/artefact/git/pull-request.md). + + * Fork main-branch sync: the push entry workflow now synchronizes the current repository's *main* from a configured source repository (default *infinito-nexus/core*) before CI scope and deploy discovery run, so a fork branch push sees only its own changes against an up-to-date base. The source is overridable or disablable through the *CI_SYNC_MAIN_SOURCE_REPOSITORY* repository variable, and the sync force-overwrites *main*, so a synced repository must not be used as a working branch. See [the Actions configuration reference](docs/contributing/tools/github/actions/configuration.md). + + * Package-changelog generator hardening: the Debian changelog and the Fedora spec *%changelog* are mirrored from CHANGELOG.md without tripping their native parsers โ€” every change line is indented for Debian, every body line gains a dash continuation for RPM, and the archived-releases notice is folded into the oldest kept entry instead of trailing after the final entry. Previously the un-indented bodies made *dpkg-buildpackage* abort the Debian package build with an empty maintainer. The *Changelog* title is also preserved when the active window is trimmed. See [the release procedure](docs/contributing/actions/release.md). + + * New packaging lint: a *lint-packages* check, wired into *make lint* and its own CI workflow, validates the generated *debian/changelog* with *dpkg-parsechangelog*, the Fedora spec with *rpmspec* and the Arch *PKGBUILD* with a *bash -n* syntax pass, failing on any parser warning rather than only on a non-zero exit. A complementary *test-lint* check asserts that both package changelogs list every released version and that their inline window matches CHANGELOG.md. See [the workflow catalog](docs/contributing/tools/github/actions/workflows.md). + + * Repository cleanup automation: a scheduled cleanup workflow plus a GHCR helper prune stale CI images so the container registry does not accumulate per-commit build artefacts. + + * Routine maintenance: YAML normalization of the *web-app-pihole* role variables (document start, single-space keys). + + **Contributors** + + * [Kevin Veen-Birkenbach](https://veen.world): trusted-fork-PR flow, fork main sync, package-changelog generator and packaging lint, cleanup automation + + -- Kevin Veen-Birkenbach Sat, 27 Jun 2026 00:00:00 +0000 + +infinito-nexus (11.0.0-1) unstable; urgency=medium + + * Unified addon syntax (requirement 026): every role-level plugin, browser/GNOME extension, Matrix bridge and cross-role integration was migrated to a single declarative *meta/addons/* contract, replacing the per-role ad-hoc plugin definitions. 86 addon files now describe addons for *desk-chromium*, *desk-firefox*, *desk-gnome*, *web-app-discourse*, *web-app-friendica*, *web-app-joomla*, *web-app-mediawiki*, *web-app-pretix*, *web-app-wordpress*, *web-app-xwiki*, *web-app-nextcloud*, *web-app-odoo* and *web-app-matrix*. Each service-coupled addon gets a *tasks/addons* integration hook and its own Playwright spec (87 per-addon specs added), addon flags are gated on actual partner deployment through a new *addon_env_flags* lookup, and a lint requires a hook plus a spec for every coupled addon. The cross-role integration matrix is regenerated from a dedicated data module (*integration_matrix_data.py*). + * New centralized API single-point-of-truth: an *api* lookup plugin plus a global API dict so API tokens and endpoints (Cloudflare, the Nextcloud proprietary OAuth integrations) resolve from one place instead of scattered role lookups. + * Real Nextcloud integration coupling: end-to-end tests now assert genuine two-way OAuth/API coupling for *gitlab*, *mattermost*, *openproject*, *zammad*, *peertube*, *discourse*, *github*, *google*, *jira*, *mastodon*, *matrix*, *moodle* and *suitecrm*, rather than the mere presence of a connect button. All integration OAuth credentials are stored Nextcloud-encrypted via ICrypto (sensitive app values), OpenProject and Zammad secrets survive a re-deploy, and OpenWebUI/Ollama is wired in as a real *integration_openai* chat backend. Bridges with no Nextcloud-33 plugin (Moodle, SuiteCRM) are disabled. + * Nextcloud deploy-variant rebalancing: Discourse and OnlyOffice are each isolated into their own loosely-coupled partner variant, the matrix is repacked to five variants overriding only the dynamic flags, LDAP is enabled wherever SSO is on, and the database is shared outside variant 1. The GitLab 19 OAuth app now carries an organization, and *fileslibreofficeedit* is gated on OnlyOffice absence. + * Storage- and runtime-aware CI variant matrix: variant bundles are split per runner by a storage-aware job splitter driven by an explicit variant CSV, behind a unified variant selector (*variant_select.py*). The default per-bundle storage cap is centralized at 350GB, per-variant memory budgets are guarded against the host budget, shared database providers are enforced by a shared-false-once lint, and the full variant matrix always runs (the earlier runtime time-cut was removed). + * Container resource governance: self-run and shared-provider services declare explicit compose mem/pids limits, container limits were right-sized across roles, and Node-heap services respect a floor via a new *node_max_old_space_size* lookup (Nextcloud whiteboard raised to 1g). A reworked *ressources* CLI emits per-variant and all-role resource summaries, the footprint tool gains a *min_storage* column, and shared services carry a *bond* integration-importance factor. + * Matrix bridges modernized: Signal, Telegram, WhatsApp and Slack bridges migrate to bridgev2 config, the dead mautrix-facebook/instagram bridges are replaced by mautrix-meta, bridge specs are gated by the deployed addon, Synapse loads the bridge registrations from a mounted directory, and the role now tracks *matrix-docker-ansible-deploy* from the spantaleev upstream. + * CA-trust and OIDC-over-self-signed-TLS hardening: the internal CA is installed into the OS bundle for PHP libcurl OIDC (*web-app-pixelfed*, *web-app-joomla*, plus a WordPress must-use plugin), the CA-trust wrapper is kept on probe timeout instead of degrading to env-only, and the CA injection is bounded so it cannot hang the handler. HSTS is now emitted on every response including non-2xx, *ansible-vault encrypt_string* reads plaintext from stdin so leading-dash values encrypt, and Mastodon allows inline *script-src-elem* for its server-rendered SPA. + * Image and dependency version jumps (net since 10.1.1): + * *web-app-erpnext*: v16.23.1 to v16.25.0 + * *web-app-gitlab*: 19.1.0-ee.0 to 19.1.1-ee.0 + * *web-app-opencloud*: 4.0.7 to 7.2.0 + * *web-app-seaweedfs*: 4.35 to 4.36 + * *web-app-matomo*: unpinned *latest* to pinned 5.3.2 (matches the bootstrap installer) + * *web-app-nextcloud* proxy (nginx): unpinned *alpine* to pinned 1.31.2-alpine + * *web-app-nextcloud* whiteboard: unpinned *latest* to pinned v1.5.9 + * *svc-ai-ollama*: switched to the official *ollama* image so inference works + * *web-app-pretix* and *web-app-xwiki*: their inline plugin/extension version pins (2.3.1; 2.19.6, 9.15.7, 1.0) were removed as those addons moved into *meta/addons* + * Dev/CI dependencies: *globals* 15.15.0 to 17.7.0, *@playwright/test* 1.61.0 to 1.61.1, *actions/cache* 5 to 6 + * Intermediate bot bumps were corrected back down: Mattermost 11.8.2 to 11.8.1 and Nextcloud 34 to 33 (kept at 33 because of plugin incompatibilities, same pin as 10.0.0) + + **Contributors** + + * [Kevin Veen-Birkenbach](https://veen.world): unified addon syntax, API single-point-of-truth, Nextcloud integrations and variant rebalancing, CI variant bundling, resource governance, Matrix bridge migration and version maintenance + + -- Kevin Veen-Birkenbach Sat, 27 Jun 2026 00:00:00 +0000 + +infinito-nexus (10.1.1-1) unstable; urgency=medium + + * Disabled the *email* service in the *web-app-pihole* role by pinning *services.email* to a static *enabled: false* / *shared: false* with the matching *dynamic-flag* and *email* nocheck suppressions, removing the now-static email overrides from *meta/variants.yml*, and adding *javascript* true/false variant pins to keep the dynamic-flag matrix-coverage guard green. + + **Contributors** + + * [Kevin Veen-Birkenbach](https://veen.world): Pi-hole email service disablement + + -- Kevin Veen-Birkenbach Wed, 24 Jun 2026 00:00:00 +0000 + +infinito-nexus (10.1.0-1) unstable; urgency=medium + + * New Pi-hole role (*web-app-pihole*) that ships the upstream Pi-hole DNS sinkhole and network-wide ad blocker, with its admin web interface gated behind *oauth2-proxy* and Keycloak RBAC and a native admin-password login (the *FTLCONF* web API password) available as a fallback variant. The upstream resolver and timezone are configurable, the role waits on the Pi-hole HTTP endpoint during deploy, and it is exercised end-to-end with Playwright covering the OAuth2, native-login and guest-access scenarios. The role ships at lifecycle beta. + * Routine maintenance: Docker image version bumps, a dependabot update of *globals* from 15.15.0 to 17.7.0, and minor *services.yml* self-provider flag corrections for ERPNext, GitLab, SeaweedFS and the Playwright end-to-end role. + + **Contributors** + + * [Prageeth Panicker](https://github.com/pragepani): Pi-hole role and its Playwright test suite + * [Kevin Veen-Birkenbach](https://veen.world): Pi-hole role hardening and review + + -- Kevin Veen-Birkenbach Tue, 23 Jun 2026 00:00:00 +0000 + +infinito-nexus (10.0.1-1) unstable; urgency=medium + + * Mark the *web-app-erpnext* role's own *erpnext* provider service with the *playwright-service-flag* self-provider suppression in *meta/services.yml*, fixing the Playwright service-flag integration guard that 10.0.0 tripped by exposing ERPNext as a consumable shared service. + + **Contributors** + + * [Kevin Veen-Birkenbach](https://veen.world): ERPNext self-provider test fix + + -- Kevin Veen-Birkenbach Fri, 19 Jun 2026 00:00:00 +0000 + +infinito-nexus (10.0.0-1) unstable; urgency=medium + + * New engine-agnostic object-store service (*web-app-seaweedfs*, backed by either *SeaweedFS* or *MinIO*) that exposes a shared S3 endpoint, now consumed across 14 web-app roles for media and asset storage. Nextcloud, Decidim, Shopware, PeerTube, Pixelfed, Taiga, Fider, Akaunting, Mobilizon and Matrix route their uploads to the central bucket, each exercised end-to-end with a SeaweedFS consumer check that requires a fresh object key per upload. + * Trusted-header single-sign-on bridges (wave 1) for *web-app-openproject*, *web-app-snipe-it*, *web-app-postmarks* and *web-app-yourls*, with *web-app-baserow* and *web-app-bookwyrm* migrating to the header bridge and dropping their LDAP configuration. The shared proxy (*sys-svc-proxy*) now strips forgeable identity headers on every location to close trusted-header injection. + * ERPNext is exposed as a consumable shared service and renders the Keycloak SSO button on */login* under gunicorn *--preload*. Nextcloud is provisioned on the SeaweedFS S3 object store with a dedicated consumer end-to-end test and has its OIDC plugins gated as mandatory; the upgrade to version 34 is deliberately not carried out because of plugin incompatibilities, so it stays pinned at 33. + * Extensive deploy and end-to-end hardening, much of it reproduced in Docker-in-Docker: native username and password login fallbacks for the no-SSO admin persona across Akaunting, Snipe-IT, YOURLS and OpenProject; OpenProject linked to the LDAP auth source for header SSO to fix a trusted-header 401, with all rails-runner Ruby extracted to dedicated files under *files/ruby/*; Shopware retries asset and theme builds against a restarting SeaweedFS; Mailu retries user creation and waits for antispam readiness before DKIM keygen; YOURLS creates its database schema on deploy; and OpenProject boots db:migrate under Ruby 4.0. + * Routine maintenance: Docker image and git-reference bumps, skills-lock refreshes, and dependabot updates (actions/checkout 6 to 7, @playwright/test 1.60.0 to 1.61.0, eslint 10.4.1 to 10.5.0). + + **Contributors** + + * [Kevin Veen-Birkenbach](https://veen.world): object-store service, trusted-header SSO bridges, role fixes and review + + -- Kevin Veen-Birkenbach Fri, 19 Jun 2026 00:00:00 +0000 + infinito-nexus (9.3.0-1) unstable; urgency=medium * New Penpot role (web-app-penpot, requirement 235) that ships the upstream Penpot design platform comprising frontend, backend, exporter and Redis at image version 2.5.4, wired into the central Keycloak via OIDC and into the central OpenLDAP. Native local-password login is available as a toggle that is disabled automatically under OIDC, with the native-login and registration flags derived from the SSO flag. A custom JVM truststore imports the Infinito self-signed CA so OIDC over TLS succeeds, and the role is exercised end-to-end with Playwright covering OIDC login, logout and project creation. The local subnet was moved to 192.168.105.192/28 to avoid a collision with the ERPNext range. -* The Matrix role gains an ansible flavor (requirement 025) built on a unified compose template, with full Docker-in-Docker isolation, host-network mode cascaded to the addons and Jitsi, a central Postgres backend sharing the MATRIX_POSTGRES variables, and a central coturn aligned to the shared coturn variables for the MASH role. A dedicated MDAD runner image (python 3.13-slim with docker 28.0.4 CLI, migration validated against v2026.05.18.0) mounts the Infinito self-signed CA into both the runner and Synapse, and bootstrap now recovers cleanly from a stale marker. -* Smaller role fixes: ERPNext bypasses the v16 setup wizard per app and clears the Frappe cache after applying social-login, LDAP and email configuration; Friendica strips a trailing slash from the base URL and makes the admin-follow Playwright test idempotent; Funkwhale starts its API with gunicorn and binds the port. -* Routine maintenance: a lint that enforces a parameterised FROM in role Dockerfiles, a re-sync of the Claude settings allow-list with upstream, git references bumped to the latest semver tags (Bluesky 1.122.0 to 1.123.0, Bookwyrm v0.8.6 to v0.8.7), and Docker image version bumps across ERPNext v15.45.0 to v16.22.0, Friendica 2026.01 to 2026.05, Funkwhale 2.0.2 to 2.0.4, GitLab 19.0.1-ee.0 to 19.0.2-ee.0, Mattermost 11.7.2 to 11.8.0, OpenTalk v1.12.0 to v1.13.1, Prometheus v0.32.1 to v0.32.2, Shopware 3.6.0 to 3.7.0, MariaDB 12.2 to 12.3 and CoreDNS 1.14.3 to 1.14.4. + * The Matrix role gains an ansible flavor (requirement 025) built on a unified compose template, with full Docker-in-Docker isolation, host-network mode cascaded to the addons and Jitsi, a central Postgres backend sharing the MATRIX_POSTGRES variables, and a central coturn aligned to the shared coturn variables for the MASH role. A dedicated MDAD runner image (python 3.13-slim with docker 28.0.4 CLI, migration validated against v2026.05.18.0) mounts the Infinito self-signed CA into both the runner and Synapse, and bootstrap now recovers cleanly from a stale marker. + * Smaller role fixes: ERPNext bypasses the v16 setup wizard per app and clears the Frappe cache after applying social-login, LDAP and email configuration; Friendica strips a trailing slash from the base URL and makes the admin-follow Playwright test idempotent; Funkwhale starts its API with gunicorn and binds the port. + * Routine maintenance: a lint that enforces a parameterised FROM in role Dockerfiles, a re-sync of the Claude settings allow-list with upstream, git references bumped to the latest semver tags (Bluesky 1.122.0 to 1.123.0, Bookwyrm v0.8.6 to v0.8.7), and Docker image version bumps across ERPNext v15.45.0 to v16.22.0, Friendica 2026.01 to 2026.05, Funkwhale 2.0.2 to 2.0.4, GitLab 19.0.1-ee.0 to 19.0.2-ee.0, Mattermost 11.7.2 to 11.8.0, OpenTalk v1.12.0 to v1.13.1, Prometheus v0.32.1 to v0.32.2, Shopware 3.6.0 to 3.7.0, MariaDB 12.2 to 12.3 and CoreDNS 1.14.3 to 1.14.4. -**Contributors** + **Contributors** -* [Evangelos Tsakoudis](https://evangelostsak.com): Penpot role and its end-to-end test suite -* [Kevin Veen-Birkenbach](https://veen.world): Matrix ansible flavor, role fixes and review + * [Evangelos Tsakoudis](https://evangelostsak.com): Penpot role and its end-to-end test suite + * [Kevin Veen-Birkenbach](https://veen.world): Matrix ansible flavor, role fixes and review -- Kevin Veen-Birkenbach Thu, 11 Jun 2026 00:00:00 +0000 infinito-nexus (9.2.0-1) unstable; urgency=medium * New ERPNext role that ships the upstream Frappe Framework v15 stack as a web-app-erpnext role โ€” backend, frontend, websocket, scheduler and two queue workers โ€” wired into the central Keycloak via a built-in Keycloak Social Login Key and into the central OpenLDAP via the LDAP Settings doctype. The three-variant matrix (SSO plus LDAP, no auth, LDAP only) is exercised end-to-end and a repo-wide lint caps Ansible task names at 120 characters. -* New Jitsi Meet role that adds an oauth2-proxy gated meeting surface plus matching LDAP-variant Playwright coverage. The spec layout is reorganised into a shared module and per-scenario test files so future scenarios can be added without touching the monolith. -* Outbound mail and Nix install resilience: the dev-nix role guards its set_fact against getent returning None on hosts without an nixbld group, and the host-stack mail health checks keep behaving cleanly on minimal images. -* Routine maintenance: Docker image version bumps including Friendica 2026.01, eslint 10.4.0 to 10.4.1, and a dockerignore re-sync after the recent claude hooks addition. + * New Jitsi Meet role that adds an oauth2-proxy gated meeting surface plus matching LDAP-variant Playwright coverage. The spec layout is reorganised into a shared module and per-scenario test files so future scenarios can be added without touching the monolith. + * Outbound mail and Nix install resilience: the dev-nix role guards its set_fact against getent returning None on hosts without an nixbld group, and the host-stack mail health checks keep behaving cleanly on minimal images. + * Routine maintenance: Docker image version bumps including Friendica 2026.01, eslint 10.4.0 to 10.4.1, and a dockerignore re-sync after the recent claude hooks addition. -Release maintained by Kevin Veen-Birkenbach, . + **Contributors** + + * Release maintained by Kevin Veen-Birkenbach, . -- Kevin Veen-Birkenbach Wed, 03 Jun 2026 00:00:00 +0000 infinito-nexus (9.1.0-1) unstable; urgency=medium - * * More robust Nix installer on Debian, Ubuntu, Fedora and CentOS systems: stale nixbld users from earlier installation attempts are cleaned up before the multi-user installer runs, and a long-standing crash on hosts where the nixbld group is absent has been resolved. -* Outbound mail health checks no longer fail silently on minimal container images: the heartbeat-email script copes with images that ship without the hostname binary, and the SSL trust file path is detected automatically across Red Hat, Debian, Ubuntu, Alpine and NixOS layouts. -* The Zammad helpdesk role completes its single-sign-on schema unification, with the variant matrix and Playwright test suite brought in line with the new shape. -* Internal infrastructure cleanups: re-synchronised .dockerignore and a shifted weekly CI schedule. + * More robust Nix installer on Debian, Ubuntu, Fedora and CentOS systems: stale nixbld users from earlier installation attempts are cleaned up before the multi-user installer runs, and a long-standing crash on hosts where the nixbld group is absent has been resolved. + * Outbound mail health checks no longer fail silently on minimal container images: the heartbeat-email script copes with images that ship without the hostname binary, and the SSL trust file path is detected automatically across Red Hat, Debian, Ubuntu, Alpine and NixOS layouts. + * The Zammad helpdesk role completes its single-sign-on schema unification, with the variant matrix and Playwright test suite brought in line with the new shape. + * Internal infrastructure cleanups: re-synchronised .dockerignore and a shifted weekly CI schedule. -Release maintained by [Kevin Veen-Birkenbach](https://veen.world). + **Contributors** + + * Release maintained by [Kevin Veen-Birkenbach](https://veen.world). -- Kevin Veen-Birkenbach Mon, 01 Jun 2026 00:00:00 +0000 @@ -50,43 +149,43 @@ infinito-nexus (9.0.0-1) unstable; urgency=medium * Collapses the parallel **oauth2** and **oidc** top-level blocks in every role *meta/services.yml* into a single *services.sso* block with a **flavor** discriminator (*oidc* | *oauth2* | *saml*, default *oidc*). Flavor-specific keys live under *sso.* (*sso.oauth2.{origin,acl,allowed_groups}*, *sso.oidc.plugin*). -**Removed** + **Removed** -* The dedicated oauth2-proxy provider role (its directory under *roles/*) is deleted; the 5 sidecar templates and the per-consumer render task fold into **web-app-keycloak** (*templates/sso_proxy/*, *tasks/sso_proxy.yml*). **web-app-keycloak** now declares **provides: sso**. -* The two mutual-exclusion guards (*tests/integration/iam/oauth2_oidc/test_mutual_exclusive.py*, *test_acl_mutual_exclusive.py*) โ€” obsolete under the single-block schema. + * The dedicated oauth2-proxy provider role (its directory under *roles/*) is deleted; the 5 sidecar templates and the per-consumer render task fold into **web-app-keycloak** (*templates/sso_proxy/*, *tasks/sso_proxy.yml*). **web-app-keycloak** now declares **provides: sso**. + * The two mutual-exclusion guards (*tests/integration/iam/oauth2_oidc/test_mutual_exclusive.py*, *test_acl_mutual_exclusive.py*) โ€” obsolete under the single-block schema. -**Added** + **Added** -* *utils/roles/applications/services/sso.py* โ€” central resolver for **is_enabled** / **is_proxy_gated** / **is_oidc_native** plus the oauth2 sub-values (*oauth2_origin_{host,port}*, *oauth2_acl*, *oauth2_allowed_groups*). -* Three lookup plugins on top of the resolver: **sso** (single-app predicate access), **sso_proxy_consumers** (enumerates oauth2-flavored consumers), **sso_oidc_plugin** (renamed from **oidc_flavor**, Nextcloud selector). -* *tests/lint/repository/no_legacy_sso_paths/* static guard against the legacy strings re-entering the source tree. + * *utils/roles/applications/services/sso.py* โ€” central resolver for **is_enabled** / **is_proxy_gated** / **is_oidc_native** plus the oauth2 sub-values (*oauth2_origin_{host,port}*, *oauth2_acl*, *oauth2_allowed_groups*). + * Three lookup plugins on top of the resolver: **sso** (single-app predicate access), **sso_proxy_consumers** (enumerates oauth2-flavored consumers), **sso_oidc_plugin** (renamed from **oidc_flavor**, Nextcloud selector). + * *tests/lint/repository/no_legacy_sso_paths/* static guard against the legacy strings re-entering the source tree. -**Renamed** + **Renamed** -* *services..ports.local.oauth2* โ†’ *ports.local.sso* -* *PORT_BANDS.local.oauth2* โ†’ *PORT_BANDS.local.sso* -* **OAUTH2_PROXY_\*** env / Jinja vars โ†’ **SSO_PROXY_\*** -* *oauth2_proxy_cookie_secret* โ†’ *sso_proxy_cookie_secret* -* **OAUTH2_SERVICE_ENABLED** + **OIDC_SERVICE_ENABLED** โ†’ single **SSO_SERVICE_ENABLED** -* Playwright gating helpers (*isServiceEnabled*, *skipUnlessServiceEnabled*, *requireService*, *safeIsEnabled*, *safeSkipUnlessEnabled*) move from service *oidc* / *oauth2* to *sso* -* *test_oauth2_contract.py* โ†’ *test_sso_contract.py* + * *services..ports.local.oauth2* โ†’ *ports.local.sso* + * *PORT_BANDS.local.oauth2* โ†’ *PORT_BANDS.local.sso* + * **OAUTH2_PROXY_\*** env / Jinja vars โ†’ **SSO_PROXY_\*** + * *oauth2_proxy_cookie_secret* โ†’ *sso_proxy_cookie_secret* + * **OAUTH2_SERVICE_ENABLED** + **OIDC_SERVICE_ENABLED** โ†’ single **SSO_SERVICE_ENABLED** + * Playwright gating helpers (*isServiceEnabled*, *skipUnlessServiceEnabled*, *requireService*, *safeIsEnabled*, *safeSkipUnlessEnabled*) move from service *oidc* / *oauth2* to *sso* + * *test_oauth2_contract.py* โ†’ *test_sso_contract.py* -**Changed** + **Changed** -* **sys-stk-backend**, **sys-svc-compose**, **sys-svc-proxy** and *plugins/filter/compose_volumes.py* now gate on **lookup(sso, โ€ฆ, is_proxy_gated)** instead of hand-combining **enabled** + **flavor**. -* Dual-block resolution: **web-app-bookwyrm** pinned to **flavor: oauth2**; **web-app-gitea** and **web-app-friendica** keep **flavor: oauth2** with the existing *oauth2.acl.blacklist* (*/user/login*) and *oauth2.acl.whitelist* (federation + discovery endpoints) โ€” operative oauth2-proxy gating preserved. + * **sys-stk-backend**, **sys-svc-compose**, **sys-svc-proxy** and *plugins/filter/compose_volumes.py* now gate on **lookup(sso, โ€ฆ, is_proxy_gated)** instead of hand-combining **enabled** + **flavor**. + * Dual-block resolution: **web-app-bookwyrm** pinned to **flavor: oauth2**; **web-app-gitea** and **web-app-friendica** keep **flavor: oauth2** with the existing *oauth2.acl.blacklist* (*/user/login*) and *oauth2.acl.whitelist* (federation + discovery endpoints) โ€” operative oauth2-proxy gating preserved. -**Migration** + **Migration** -Every consumer of the legacy oauth2 / oidc service sub-trees, the **OAUTH2_PROXY_\*** env / Jinja vars, **OAUTH2_SERVICE_ENABLED**, **OIDC_SERVICE_ENABLED**, *oauth2_proxy_cookie_secret*, *oauth2_proxy_application_id*, *ports.local.oauth2*, the dedicated oauth2-proxy provider role, or the legacy *oidc* / *oauth2* Playwright gating helper service names must migrate to the unified **sso** shape. Full plan and decisions are archived under req 21. + Every consumer of the legacy oauth2 / oidc service sub-trees, the **OAUTH2_PROXY_\*** env / Jinja vars, **OAUTH2_SERVICE_ENABLED**, **OIDC_SERVICE_ENABLED**, *oauth2_proxy_cookie_secret*, *oauth2_proxy_application_id*, *ports.local.oauth2*, the dedicated oauth2-proxy provider role, or the legacy *oidc* / *oauth2* Playwright gating helper service names must migrate to the unified **sso** shape. Full plan and decisions are archived under req 21. -**Validated** + **Validated** -End-to-end against *web-app-{keycloak,nextcloud,bookwyrm,gitea,friendica,dashboard}* in matrix variants 0 and 1; v2 also passes Playwright for the four target roles (modulo a pre-existing **KC_HOSTNAME**-drift workaround, separate concern). + End-to-end against *web-app-{keycloak,nextcloud,bookwyrm,gitea,friendica,dashboard}* in matrix variants 0 and 1; v2 also passes Playwright for the four target roles (modulo a pre-existing **KC_HOSTNAME**-drift workaround, separate concern). -**Contributors** + **Contributors** -* [Kevin Veen-Birkenbach](https://www.veen.world/) + * [Kevin Veen-Birkenbach](https://www.veen.world/) -- Kevin Veen-Birkenbach Thu, 28 May 2026 00:00:00 +0000 @@ -94,57 +193,57 @@ infinito-nexus (8.0.5-1) unstable; urgency=medium * Adds --version / -V flag to the infinito CLI, drops a dead Compose-CLI reference link, and fixes the dev-deploy router so per-invocation routing knobs (apps, mode, purge, bundles, disable, full_cycle, variant) no longer leak into the persistent .env and the make-alias / env-var surface is unified 1:1. + For older releases, see https://docs.infinito.nexus/ + + Earlier releases: + 8.0.4 (2026-05-28) + 8.0.3 (2026-05-28) + 8.0.2 (2026-05-28) + 8.0.1 (2026-05-28) + 8.0.0 (2026-05-27) + 7.0.0 (2026-05-08) + 6.0.0 (2026-04-25) + 5.2.0 (2026-03-21) + 5.1.0 (2026-02-28) + 5.0.0 (2026-02-25) + 4.1.0 (2026-02-17) + 4.0.3 (2026-02-16) + 4.0.2 (2026-02-15) + 4.0.1 (2026-02-15) + 4.0.0 (2026-02-13) + 3.0.0 (2026-02-11) + 2.1.9 (2026-02-10) + 2.1.8 (2026-02-09) + 2.1.7 (2026-02-09) + 2.1.6 (2026-02-09) + 2.1.5 (2026-02-09) + 2.1.4 (2026-02-08) + 2.1.3 (2026-02-08) + 2.1.2 (2026-02-08) + 2.1.1 (2026-02-08) + 2.1.0 (2026-02-08) + 2.0.0 (2026-02-08) + 1.0.0 (2026-02-03) + 0.12.0 (2026-01-25) + 0.11.0 (2026-01-10) + 0.10.0 (2026-01-08) + 0.9.0 (2026-01-07) + 0.8.0 (2026-01-06) + 0.7.2 (2026-01-06) + 0.7.1 (2026-01-06) + 0.7.0 (2026-01-05) + 0.6.0 (2025-12-31) + 0.5.0 (2025-12-30) + 0.4.0 (2025-12-29) + 0.3.5 (2025-12-21) + 0.3.4 (2025-12-21) + 0.3.3 (2025-12-21) + 0.3.2 (2025-12-19) + 0.3.1 (2025-12-18) + 0.3.0 (2025-12-17) + 0.2.1 (2025-12-10) + 0.2.0 (2025-12-10) + 0.1.1 (2025-12-10) + 0.1.0 (2025-12-09) + -- Kevin Veen-Birkenbach Thu, 28 May 2026 00:00:00 +0000 - -For older releases, see https://docs.infinito.nexus/ - -Earlier releases: - 8.0.4 (2026-05-28) - 8.0.3 (2026-05-28) - 8.0.2 (2026-05-28) - 8.0.1 (2026-05-28) - 8.0.0 (2026-05-27) - 7.0.0 (2026-05-08) - 6.0.0 (2026-04-25) - 5.2.0 (2026-03-21) - 5.1.0 (2026-02-28) - 5.0.0 (2026-02-25) - 4.1.0 (2026-02-17) - 4.0.3 (2026-02-16) - 4.0.2 (2026-02-15) - 4.0.1 (2026-02-15) - 4.0.0 (2026-02-13) - 3.0.0 (2026-02-11) - 2.1.9 (2026-02-10) - 2.1.8 (2026-02-09) - 2.1.7 (2026-02-09) - 2.1.6 (2026-02-09) - 2.1.5 (2026-02-09) - 2.1.4 (2026-02-08) - 2.1.3 (2026-02-08) - 2.1.2 (2026-02-08) - 2.1.1 (2026-02-08) - 2.1.0 (2026-02-08) - 2.0.0 (2026-02-08) - 1.0.0 (2026-02-03) - 0.12.0 (2026-01-25) - 0.11.0 (2026-01-10) - 0.10.0 (2026-01-08) - 0.9.0 (2026-01-07) - 0.8.0 (2026-01-06) - 0.7.2 (2026-01-06) - 0.7.1 (2026-01-06) - 0.7.0 (2026-01-05) - 0.6.0 (2025-12-31) - 0.5.0 (2025-12-30) - 0.4.0 (2025-12-29) - 0.3.5 (2025-12-21) - 0.3.4 (2025-12-21) - 0.3.3 (2025-12-21) - 0.3.2 (2025-12-19) - 0.3.1 (2025-12-18) - 0.3.0 (2025-12-17) - 0.2.1 (2025-12-10) - 0.2.0 (2025-12-10) - 0.1.1 (2025-12-10) - 0.1.0 (2025-12-09) diff --git a/packaging/fedora/infinito-nexus.spec b/packaging/fedora/infinito-nexus.spec index 70cf9e991d..a88a50618b 100644 --- a/packaging/fedora/infinito-nexus.spec +++ b/packaging/fedora/infinito-nexus.spec @@ -1,5 +1,5 @@ Name: infinito-nexus -Version: 9.3.0 +Version: 11.1.0 Release: 1%{?dist} Summary: Meta package for Infinito.Nexus host dependencies @@ -58,32 +58,113 @@ install -d %{buildroot}%{_docdir}/%{name} %doc %{_docdir}/%{name}/DEPENDENCIES %changelog +* Sat Jun 27 2026 Kevin Veen-Birkenbach - 11.1.0-1 +- * Trusted fork PRs: a maintainer-applied *trusted-pr* label runs a fork PR as deliberately released code โ€” the orchestrator skips the fork-prerequisite wait and the privileged *pull_request_target* build switches its source from the base merge ref to the raw PR head, while the built image keeps the merge-SHA tag so every downstream consumer is unchanged. The label is the only switch (applying a label is collaborator-only on GitHub, so fork authors cannot grant it), build-orchestration scripts always come from the base repository, and both unlabeled fork PRs and same-repository PRs are untouched. See [the fork-PR pipeline](docs/contributing/artefact/git/pipeline.md) and [the contributor fork workflow](docs/contributing/artefact/git/pull-request.md). + +- * Fork main-branch sync: the push entry workflow now synchronizes the current repository's *main* from a configured source repository (default *infinito-nexus/core*) before CI scope and deploy discovery run, so a fork branch push sees only its own changes against an up-to-date base. The source is overridable or disablable through the *CI_SYNC_MAIN_SOURCE_REPOSITORY* repository variable, and the sync force-overwrites *main*, so a synced repository must not be used as a working branch. See [the Actions configuration reference](docs/contributing/tools/github/actions/configuration.md). + +- * Package-changelog generator hardening: the Debian changelog and the Fedora spec *%changelog* are mirrored from CHANGELOG.md without tripping their native parsers โ€” every change line is indented for Debian, every body line gains a dash continuation for RPM, and the archived-releases notice is folded into the oldest kept entry instead of trailing after the final entry. Previously the un-indented bodies made *dpkg-buildpackage* abort the Debian package build with an empty maintainer. The *Changelog* title is also preserved when the active window is trimmed. See [the release procedure](docs/contributing/actions/release.md). + +- * New packaging lint: a *lint-packages* check, wired into *make lint* and its own CI workflow, validates the generated *debian/changelog* with *dpkg-parsechangelog*, the Fedora spec with *rpmspec* and the Arch *PKGBUILD* with a *bash -n* syntax pass, failing on any parser warning rather than only on a non-zero exit. A complementary *test-lint* check asserts that both package changelogs list every released version and that their inline window matches CHANGELOG.md. See [the workflow catalog](docs/contributing/tools/github/actions/workflows.md). + +- * Repository cleanup automation: a scheduled cleanup workflow plus a GHCR helper prune stale CI images so the container registry does not accumulate per-commit build artefacts. + +- * Routine maintenance: YAML normalization of the *web-app-pihole* role variables (document start, single-space keys). + +- **Contributors** + +- * [Kevin Veen-Birkenbach](https://veen.world): trusted-fork-PR flow, fork main sync, package-changelog generator and packaging lint, cleanup automation + +* Sat Jun 27 2026 Kevin Veen-Birkenbach - 11.0.0-1 +- * Unified addon syntax (requirement 026): every role-level plugin, browser/GNOME extension, Matrix bridge and cross-role integration was migrated to a single declarative *meta/addons/* contract, replacing the per-role ad-hoc plugin definitions. 86 addon files now describe addons for *desk-chromium*, *desk-firefox*, *desk-gnome*, *web-app-discourse*, *web-app-friendica*, *web-app-joomla*, *web-app-mediawiki*, *web-app-pretix*, *web-app-wordpress*, *web-app-xwiki*, *web-app-nextcloud*, *web-app-odoo* and *web-app-matrix*. Each service-coupled addon gets a *tasks/addons* integration hook and its own Playwright spec (87 per-addon specs added), addon flags are gated on actual partner deployment through a new *addon_env_flags* lookup, and a lint requires a hook plus a spec for every coupled addon. The cross-role integration matrix is regenerated from a dedicated data module (*integration_matrix_data.py*). +- * New centralized API single-point-of-truth: an *api* lookup plugin plus a global API dict so API tokens and endpoints (Cloudflare, the Nextcloud proprietary OAuth integrations) resolve from one place instead of scattered role lookups. +- * Real Nextcloud integration coupling: end-to-end tests now assert genuine two-way OAuth/API coupling for *gitlab*, *mattermost*, *openproject*, *zammad*, *peertube*, *discourse*, *github*, *google*, *jira*, *mastodon*, *matrix*, *moodle* and *suitecrm*, rather than the mere presence of a connect button. All integration OAuth credentials are stored Nextcloud-encrypted via ICrypto (sensitive app values), OpenProject and Zammad secrets survive a re-deploy, and OpenWebUI/Ollama is wired in as a real *integration_openai* chat backend. Bridges with no Nextcloud-33 plugin (Moodle, SuiteCRM) are disabled. +- * Nextcloud deploy-variant rebalancing: Discourse and OnlyOffice are each isolated into their own loosely-coupled partner variant, the matrix is repacked to five variants overriding only the dynamic flags, LDAP is enabled wherever SSO is on, and the database is shared outside variant 1. The GitLab 19 OAuth app now carries an organization, and *fileslibreofficeedit* is gated on OnlyOffice absence. +- * Storage- and runtime-aware CI variant matrix: variant bundles are split per runner by a storage-aware job splitter driven by an explicit variant CSV, behind a unified variant selector (*variant_select.py*). The default per-bundle storage cap is centralized at 350GB, per-variant memory budgets are guarded against the host budget, shared database providers are enforced by a shared-false-once lint, and the full variant matrix always runs (the earlier runtime time-cut was removed). +- * Container resource governance: self-run and shared-provider services declare explicit compose mem/pids limits, container limits were right-sized across roles, and Node-heap services respect a floor via a new *node_max_old_space_size* lookup (Nextcloud whiteboard raised to 1g). A reworked *ressources* CLI emits per-variant and all-role resource summaries, the footprint tool gains a *min_storage* column, and shared services carry a *bond* integration-importance factor. +- * Matrix bridges modernized: Signal, Telegram, WhatsApp and Slack bridges migrate to bridgev2 config, the dead mautrix-facebook/instagram bridges are replaced by mautrix-meta, bridge specs are gated by the deployed addon, Synapse loads the bridge registrations from a mounted directory, and the role now tracks *matrix-docker-ansible-deploy* from the spantaleev upstream. +- * CA-trust and OIDC-over-self-signed-TLS hardening: the internal CA is installed into the OS bundle for PHP libcurl OIDC (*web-app-pixelfed*, *web-app-joomla*, plus a WordPress must-use plugin), the CA-trust wrapper is kept on probe timeout instead of degrading to env-only, and the CA injection is bounded so it cannot hang the handler. HSTS is now emitted on every response including non-2xx, *ansible-vault encrypt_string* reads plaintext from stdin so leading-dash values encrypt, and Mastodon allows inline *script-src-elem* for its server-rendered SPA. +- * Image and dependency version jumps (net since 10.1.1): +- * *web-app-erpnext*: v16.23.1 to v16.25.0 +- * *web-app-gitlab*: 19.1.0-ee.0 to 19.1.1-ee.0 +- * *web-app-opencloud*: 4.0.7 to 7.2.0 +- * *web-app-seaweedfs*: 4.35 to 4.36 +- * *web-app-matomo*: unpinned *latest* to pinned 5.3.2 (matches the bootstrap installer) +- * *web-app-nextcloud* proxy (nginx): unpinned *alpine* to pinned 1.31.2-alpine +- * *web-app-nextcloud* whiteboard: unpinned *latest* to pinned v1.5.9 +- * *svc-ai-ollama*: switched to the official *ollama* image so inference works +- * *web-app-pretix* and *web-app-xwiki*: their inline plugin/extension version pins (2.3.1; 2.19.6, 9.15.7, 1.0) were removed as those addons moved into *meta/addons* +- * Dev/CI dependencies: *globals* 15.15.0 to 17.7.0, *@playwright/test* 1.61.0 to 1.61.1, *actions/cache* 5 to 6 +- * Intermediate bot bumps were corrected back down: Mattermost 11.8.2 to 11.8.1 and Nextcloud 34 to 33 (kept at 33 because of plugin incompatibilities, same pin as 10.0.0) + +- **Contributors** + +- * [Kevin Veen-Birkenbach](https://veen.world): unified addon syntax, API single-point-of-truth, Nextcloud integrations and variant rebalancing, CI variant bundling, resource governance, Matrix bridge migration and version maintenance + +* Wed Jun 24 2026 Kevin Veen-Birkenbach - 10.1.1-1 +- * Disabled the *email* service in the *web-app-pihole* role by pinning *services.email* to a static *enabled: false* / *shared: false* with the matching *dynamic-flag* and *email* nocheck suppressions, removing the now-static email overrides from *meta/variants.yml*, and adding *javascript* true/false variant pins to keep the dynamic-flag matrix-coverage guard green. + +- **Contributors** + +- * [Kevin Veen-Birkenbach](https://veen.world): Pi-hole email service disablement + +* Tue Jun 23 2026 Kevin Veen-Birkenbach - 10.1.0-1 +- * New Pi-hole role (*web-app-pihole*) that ships the upstream Pi-hole DNS sinkhole and network-wide ad blocker, with its admin web interface gated behind *oauth2-proxy* and Keycloak RBAC and a native admin-password login (the *FTLCONF* web API password) available as a fallback variant. The upstream resolver and timezone are configurable, the role waits on the Pi-hole HTTP endpoint during deploy, and it is exercised end-to-end with Playwright covering the OAuth2, native-login and guest-access scenarios. The role ships at lifecycle beta. +- * Routine maintenance: Docker image version bumps, a dependabot update of *globals* from 15.15.0 to 17.7.0, and minor *services.yml* self-provider flag corrections for ERPNext, GitLab, SeaweedFS and the Playwright end-to-end role. + +- **Contributors** + +- * [Prageeth Panicker](https://github.com/pragepani): Pi-hole role and its Playwright test suite +- * [Kevin Veen-Birkenbach](https://veen.world): Pi-hole role hardening and review + +* Fri Jun 19 2026 Kevin Veen-Birkenbach - 10.0.1-1 +- * Mark the *web-app-erpnext* role's own *erpnext* provider service with the *playwright-service-flag* self-provider suppression in *meta/services.yml*, fixing the Playwright service-flag integration guard that 10.0.0 tripped by exposing ERPNext as a consumable shared service. + +- **Contributors** + +- * [Kevin Veen-Birkenbach](https://veen.world): ERPNext self-provider test fix + +* Fri Jun 19 2026 Kevin Veen-Birkenbach - 10.0.0-1 +- * New engine-agnostic object-store service (*web-app-seaweedfs*, backed by either *SeaweedFS* or *MinIO*) that exposes a shared S3 endpoint, now consumed across 14 web-app roles for media and asset storage. Nextcloud, Decidim, Shopware, PeerTube, Pixelfed, Taiga, Fider, Akaunting, Mobilizon and Matrix route their uploads to the central bucket, each exercised end-to-end with a SeaweedFS consumer check that requires a fresh object key per upload. +- * Trusted-header single-sign-on bridges (wave 1) for *web-app-openproject*, *web-app-snipe-it*, *web-app-postmarks* and *web-app-yourls*, with *web-app-baserow* and *web-app-bookwyrm* migrating to the header bridge and dropping their LDAP configuration. The shared proxy (*sys-svc-proxy*) now strips forgeable identity headers on every location to close trusted-header injection. +- * ERPNext is exposed as a consumable shared service and renders the Keycloak SSO button on */login* under gunicorn *--preload*. Nextcloud is provisioned on the SeaweedFS S3 object store with a dedicated consumer end-to-end test and has its OIDC plugins gated as mandatory; the upgrade to version 34 is deliberately not carried out because of plugin incompatibilities, so it stays pinned at 33. +- * Extensive deploy and end-to-end hardening, much of it reproduced in Docker-in-Docker: native username and password login fallbacks for the no-SSO admin persona across Akaunting, Snipe-IT, YOURLS and OpenProject; OpenProject linked to the LDAP auth source for header SSO to fix a trusted-header 401, with all rails-runner Ruby extracted to dedicated files under *files/ruby/*; Shopware retries asset and theme builds against a restarting SeaweedFS; Mailu retries user creation and waits for antispam readiness before DKIM keygen; YOURLS creates its database schema on deploy; and OpenProject boots db:migrate under Ruby 4.0. +- * Routine maintenance: Docker image and git-reference bumps, skills-lock refreshes, and dependabot updates (actions/checkout 6 to 7, @playwright/test 1.60.0 to 1.61.0, eslint 10.4.1 to 10.5.0). + +- **Contributors** + +- * [Kevin Veen-Birkenbach](https://veen.world): object-store service, trusted-header SSO bridges, role fixes and review + * Thu Jun 11 2026 Kevin Veen-Birkenbach - 9.3.0-1 - * New Penpot role (web-app-penpot, requirement 235) that ships the upstream Penpot design platform comprising frontend, backend, exporter and Redis at image version 2.5.4, wired into the central Keycloak via OIDC and into the central OpenLDAP. Native local-password login is available as a toggle that is disabled automatically under OIDC, with the native-login and registration flags derived from the SSO flag. A custom JVM truststore imports the Infinito self-signed CA so OIDC over TLS succeeds, and the role is exercised end-to-end with Playwright covering OIDC login, logout and project creation. The local subnet was moved to 192.168.105.192/28 to avoid a collision with the ERPNext range. -* The Matrix role gains an ansible flavor (requirement 025) built on a unified compose template, with full Docker-in-Docker isolation, host-network mode cascaded to the addons and Jitsi, a central Postgres backend sharing the MATRIX_POSTGRES variables, and a central coturn aligned to the shared coturn variables for the MASH role. A dedicated MDAD runner image (python 3.13-slim with docker 28.0.4 CLI, migration validated against v2026.05.18.0) mounts the Infinito self-signed CA into both the runner and Synapse, and bootstrap now recovers cleanly from a stale marker. -* Smaller role fixes: ERPNext bypasses the v16 setup wizard per app and clears the Frappe cache after applying social-login, LDAP and email configuration; Friendica strips a trailing slash from the base URL and makes the admin-follow Playwright test idempotent; Funkwhale starts its API with gunicorn and binds the port. -* Routine maintenance: a lint that enforces a parameterised FROM in role Dockerfiles, a re-sync of the Claude settings allow-list with upstream, git references bumped to the latest semver tags (Bluesky 1.122.0 to 1.123.0, Bookwyrm v0.8.6 to v0.8.7), and Docker image version bumps across ERPNext v15.45.0 to v16.22.0, Friendica 2026.01 to 2026.05, Funkwhale 2.0.2 to 2.0.4, GitLab 19.0.1-ee.0 to 19.0.2-ee.0, Mattermost 11.7.2 to 11.8.0, OpenTalk v1.12.0 to v1.13.1, Prometheus v0.32.1 to v0.32.2, Shopware 3.6.0 to 3.7.0, MariaDB 12.2 to 12.3 and CoreDNS 1.14.3 to 1.14.4. +- * The Matrix role gains an ansible flavor (requirement 025) built on a unified compose template, with full Docker-in-Docker isolation, host-network mode cascaded to the addons and Jitsi, a central Postgres backend sharing the MATRIX_POSTGRES variables, and a central coturn aligned to the shared coturn variables for the MASH role. A dedicated MDAD runner image (python 3.13-slim with docker 28.0.4 CLI, migration validated against v2026.05.18.0) mounts the Infinito self-signed CA into both the runner and Synapse, and bootstrap now recovers cleanly from a stale marker. +- * Smaller role fixes: ERPNext bypasses the v16 setup wizard per app and clears the Frappe cache after applying social-login, LDAP and email configuration; Friendica strips a trailing slash from the base URL and makes the admin-follow Playwright test idempotent; Funkwhale starts its API with gunicorn and binds the port. +- * Routine maintenance: a lint that enforces a parameterised FROM in role Dockerfiles, a re-sync of the Claude settings allow-list with upstream, git references bumped to the latest semver tags (Bluesky 1.122.0 to 1.123.0, Bookwyrm v0.8.6 to v0.8.7), and Docker image version bumps across ERPNext v15.45.0 to v16.22.0, Friendica 2026.01 to 2026.05, Funkwhale 2.0.2 to 2.0.4, GitLab 19.0.1-ee.0 to 19.0.2-ee.0, Mattermost 11.7.2 to 11.8.0, OpenTalk v1.12.0 to v1.13.1, Prometheus v0.32.1 to v0.32.2, Shopware 3.6.0 to 3.7.0, MariaDB 12.2 to 12.3 and CoreDNS 1.14.3 to 1.14.4. -**Contributors** +- **Contributors** -* [Evangelos Tsakoudis](https://evangelostsak.com): Penpot role and its end-to-end test suite -* [Kevin Veen-Birkenbach](https://veen.world): Matrix ansible flavor, role fixes and review +- * [Evangelos Tsakoudis](https://evangelostsak.com): Penpot role and its end-to-end test suite +- * [Kevin Veen-Birkenbach](https://veen.world): Matrix ansible flavor, role fixes and review * Wed Jun 03 2026 Kevin Veen-Birkenbach - 9.2.0-1 - * New ERPNext role that ships the upstream Frappe Framework v15 stack as a web-app-erpnext role โ€” backend, frontend, websocket, scheduler and two queue workers โ€” wired into the central Keycloak via a built-in Keycloak Social Login Key and into the central OpenLDAP via the LDAP Settings doctype. The three-variant matrix (SSO plus LDAP, no auth, LDAP only) is exercised end-to-end and a repo-wide lint caps Ansible task names at 120 characters. -* New Jitsi Meet role that adds an oauth2-proxy gated meeting surface plus matching LDAP-variant Playwright coverage. The spec layout is reorganised into a shared module and per-scenario test files so future scenarios can be added without touching the monolith. -* Outbound mail and Nix install resilience: the dev-nix role guards its set_fact against getent returning None on hosts without an nixbld group, and the host-stack mail health checks keep behaving cleanly on minimal images. -* Routine maintenance: Docker image version bumps including Friendica 2026.01, eslint 10.4.0 to 10.4.1, and a dockerignore re-sync after the recent claude hooks addition. +- * New Jitsi Meet role that adds an oauth2-proxy gated meeting surface plus matching LDAP-variant Playwright coverage. The spec layout is reorganised into a shared module and per-scenario test files so future scenarios can be added without touching the monolith. +- * Outbound mail and Nix install resilience: the dev-nix role guards its set_fact against getent returning None on hosts without an nixbld group, and the host-stack mail health checks keep behaving cleanly on minimal images. +- * Routine maintenance: Docker image version bumps including Friendica 2026.01, eslint 10.4.0 to 10.4.1, and a dockerignore re-sync after the recent claude hooks addition. -Release maintained by Kevin Veen-Birkenbach, . +- **Contributors** + +- * Release maintained by Kevin Veen-Birkenbach, . * Mon Jun 01 2026 Kevin Veen-Birkenbach - 9.1.0-1 -- * * More robust Nix installer on Debian, Ubuntu, Fedora and CentOS systems: stale nixbld users from earlier installation attempts are cleaned up before the multi-user installer runs, and a long-standing crash on hosts where the nixbld group is absent has been resolved. -* Outbound mail health checks no longer fail silently on minimal container images: the heartbeat-email script copes with images that ship without the hostname binary, and the SSL trust file path is detected automatically across Red Hat, Debian, Ubuntu, Alpine and NixOS layouts. -* The Zammad helpdesk role completes its single-sign-on schema unification, with the variant matrix and Playwright test suite brought in line with the new shape. -* Internal infrastructure cleanups: re-synchronised .dockerignore and a shifted weekly CI schedule. +- * More robust Nix installer on Debian, Ubuntu, Fedora and CentOS systems: stale nixbld users from earlier installation attempts are cleaned up before the multi-user installer runs, and a long-standing crash on hosts where the nixbld group is absent has been resolved. +- * Outbound mail health checks no longer fail silently on minimal container images: the heartbeat-email script copes with images that ship without the hostname binary, and the SSL trust file path is detected automatically across Red Hat, Debian, Ubuntu, Alpine and NixOS layouts. +- * The Zammad helpdesk role completes its single-sign-on schema unification, with the variant matrix and Playwright test suite brought in line with the new shape. +- * Internal infrastructure cleanups: re-synchronised .dockerignore and a shifted weekly CI schedule. -Release maintained by [Kevin Veen-Birkenbach](https://veen.world). +- **Contributors** + +- * Release maintained by [Kevin Veen-Birkenbach](https://veen.world). * Fri May 29 2026 Kevin Veen-Birkenbach - 9.0.2-1 - * Restores the Debian dev-image build by introducing the INFINITO_VENV_DIR SPOT and calling infinito via its absolute venv path in scripts/docker/entry.sh, so the post-`make install` version check no longer trips on the PATH that bash -lc clobbers via /etc/profile; also scopes the auto-update PR dedup fingerprint to the files a run actually committed (instead of the whole commit tree) so unrelated drift on main no longer forces a fresh PR on every daily cron, nests web-app-bluesky's nocheck markers under the acl block they describe, and rolls up dependabot bumps (actions/cache 4โ†’5, actions/setup-node 4โ†’6) plus routine Docker image / git ref refreshes. @@ -94,96 +175,96 @@ Release maintained by [Kevin Veen-Birkenbach](https://veen.world). * Thu May 28 2026 Kevin Veen-Birkenbach - 9.0.0-1 - * Collapses the parallel **oauth2** and **oidc** top-level blocks in every role *meta/services.yml* into a single *services.sso* block with a **flavor** discriminator (*oidc* | *oauth2* | *saml*, default *oidc*). Flavor-specific keys live under *sso.* (*sso.oauth2.{origin,acl,allowed_groups}*, *sso.oidc.plugin*). -**Removed** +- **Removed** -* The dedicated oauth2-proxy provider role (its directory under *roles/*) is deleted; the 5 sidecar templates and the per-consumer render task fold into **web-app-keycloak** (*templates/sso_proxy/*, *tasks/sso_proxy.yml*). **web-app-keycloak** now declares **provides: sso**. -* The two mutual-exclusion guards (*tests/integration/iam/oauth2_oidc/test_mutual_exclusive.py*, *test_acl_mutual_exclusive.py*) โ€” obsolete under the single-block schema. +- * The dedicated oauth2-proxy provider role (its directory under *roles/*) is deleted; the 5 sidecar templates and the per-consumer render task fold into **web-app-keycloak** (*templates/sso_proxy/*, *tasks/sso_proxy.yml*). **web-app-keycloak** now declares **provides: sso**. +- * The two mutual-exclusion guards (*tests/integration/iam/oauth2_oidc/test_mutual_exclusive.py*, *test_acl_mutual_exclusive.py*) โ€” obsolete under the single-block schema. -**Added** +- **Added** -* *utils/roles/applications/services/sso.py* โ€” central resolver for **is_enabled** / **is_proxy_gated** / **is_oidc_native** plus the oauth2 sub-values (*oauth2_origin_{host,port}*, *oauth2_acl*, *oauth2_allowed_groups*). -* Three lookup plugins on top of the resolver: **sso** (single-app predicate access), **sso_proxy_consumers** (enumerates oauth2-flavored consumers), **sso_oidc_plugin** (renamed from **oidc_flavor**, Nextcloud selector). -* *tests/lint/repository/no_legacy_sso_paths/* static guard against the legacy strings re-entering the source tree. +- * *utils/roles/applications/services/sso.py* โ€” central resolver for **is_enabled** / **is_proxy_gated** / **is_oidc_native** plus the oauth2 sub-values (*oauth2_origin_{host,port}*, *oauth2_acl*, *oauth2_allowed_groups*). +- * Three lookup plugins on top of the resolver: **sso** (single-app predicate access), **sso_proxy_consumers** (enumerates oauth2-flavored consumers), **sso_oidc_plugin** (renamed from **oidc_flavor**, Nextcloud selector). +- * *tests/lint/repository/no_legacy_sso_paths/* static guard against the legacy strings re-entering the source tree. -**Renamed** +- **Renamed** -* *services..ports.local.oauth2* โ†’ *ports.local.sso* -* *PORT_BANDS.local.oauth2* โ†’ *PORT_BANDS.local.sso* -* **OAUTH2_PROXY_\*** env / Jinja vars โ†’ **SSO_PROXY_\*** -* *oauth2_proxy_cookie_secret* โ†’ *sso_proxy_cookie_secret* -* **OAUTH2_SERVICE_ENABLED** + **OIDC_SERVICE_ENABLED** โ†’ single **SSO_SERVICE_ENABLED** -* Playwright gating helpers (*isServiceEnabled*, *skipUnlessServiceEnabled*, *requireService*, *safeIsEnabled*, *safeSkipUnlessEnabled*) move from service *oidc* / *oauth2* to *sso* -* *test_oauth2_contract.py* โ†’ *test_sso_contract.py* +- * *services..ports.local.oauth2* โ†’ *ports.local.sso* +- * *PORT_BANDS.local.oauth2* โ†’ *PORT_BANDS.local.sso* +- * **OAUTH2_PROXY_\*** env / Jinja vars โ†’ **SSO_PROXY_\*** +- * *oauth2_proxy_cookie_secret* โ†’ *sso_proxy_cookie_secret* +- * **OAUTH2_SERVICE_ENABLED** + **OIDC_SERVICE_ENABLED** โ†’ single **SSO_SERVICE_ENABLED** +- * Playwright gating helpers (*isServiceEnabled*, *skipUnlessServiceEnabled*, *requireService*, *safeIsEnabled*, *safeSkipUnlessEnabled*) move from service *oidc* / *oauth2* to *sso* +- * *test_oauth2_contract.py* โ†’ *test_sso_contract.py* -**Changed** +- **Changed** -* **sys-stk-backend**, **sys-svc-compose**, **sys-svc-proxy** and *plugins/filter/compose_volumes.py* now gate on **lookup(sso, โ€ฆ, is_proxy_gated)** instead of hand-combining **enabled** + **flavor**. -* Dual-block resolution: **web-app-bookwyrm** pinned to **flavor: oauth2**; **web-app-gitea** and **web-app-friendica** keep **flavor: oauth2** with the existing *oauth2.acl.blacklist* (*/user/login*) and *oauth2.acl.whitelist* (federation + discovery endpoints) โ€” operative oauth2-proxy gating preserved. +- * **sys-stk-backend**, **sys-svc-compose**, **sys-svc-proxy** and *plugins/filter/compose_volumes.py* now gate on **lookup(sso, โ€ฆ, is_proxy_gated)** instead of hand-combining **enabled** + **flavor**. +- * Dual-block resolution: **web-app-bookwyrm** pinned to **flavor: oauth2**; **web-app-gitea** and **web-app-friendica** keep **flavor: oauth2** with the existing *oauth2.acl.blacklist* (*/user/login*) and *oauth2.acl.whitelist* (federation + discovery endpoints) โ€” operative oauth2-proxy gating preserved. -**Migration** +- **Migration** -Every consumer of the legacy oauth2 / oidc service sub-trees, the **OAUTH2_PROXY_\*** env / Jinja vars, **OAUTH2_SERVICE_ENABLED**, **OIDC_SERVICE_ENABLED**, *oauth2_proxy_cookie_secret*, *oauth2_proxy_application_id*, *ports.local.oauth2*, the dedicated oauth2-proxy provider role, or the legacy *oidc* / *oauth2* Playwright gating helper service names must migrate to the unified **sso** shape. Full plan and decisions are archived under req 21. +- Every consumer of the legacy oauth2 / oidc service sub-trees, the **OAUTH2_PROXY_\*** env / Jinja vars, **OAUTH2_SERVICE_ENABLED**, **OIDC_SERVICE_ENABLED**, *oauth2_proxy_cookie_secret*, *oauth2_proxy_application_id*, *ports.local.oauth2*, the dedicated oauth2-proxy provider role, or the legacy *oidc* / *oauth2* Playwright gating helper service names must migrate to the unified **sso** shape. Full plan and decisions are archived under req 21. -**Validated** +- **Validated** -End-to-end against *web-app-{keycloak,nextcloud,bookwyrm,gitea,friendica,dashboard}* in matrix variants 0 and 1; v2 also passes Playwright for the four target roles (modulo a pre-existing **KC_HOSTNAME**-drift workaround, separate concern). +- End-to-end against *web-app-{keycloak,nextcloud,bookwyrm,gitea,friendica,dashboard}* in matrix variants 0 and 1; v2 also passes Playwright for the four target roles (modulo a pre-existing **KC_HOSTNAME**-drift workaround, separate concern). -**Contributors** +- **Contributors** -* [Kevin Veen-Birkenbach](https://www.veen.world/) +- * [Kevin Veen-Birkenbach](https://www.veen.world/) * Thu May 28 2026 Kevin Veen-Birkenbach - 8.0.5-1 - * Adds --version / -V flag to the infinito CLI, drops a dead Compose-CLI reference link, and fixes the dev-deploy router so per-invocation routing knobs (apps, mode, purge, bundles, disable, full_cycle, variant) no longer leak into the persistent .env and the make-alias / env-var surface is unified 1:1. -For older releases, see https://docs.infinito.nexus/ +- For older releases, see https://docs.infinito.nexus/ -Earlier releases: - 8.0.4 (2026-05-28) - 8.0.3 (2026-05-28) - 8.0.2 (2026-05-28) - 8.0.1 (2026-05-28) - 8.0.0 (2026-05-27) - 7.0.0 (2026-05-08) - 6.0.0 (2026-04-25) - 5.2.0 (2026-03-21) - 5.1.0 (2026-02-28) - 5.0.0 (2026-02-25) - 4.1.0 (2026-02-17) - 4.0.3 (2026-02-16) - 4.0.2 (2026-02-15) - 4.0.1 (2026-02-15) - 4.0.0 (2026-02-13) - 3.0.0 (2026-02-11) - 2.1.9 (2026-02-10) - 2.1.8 (2026-02-09) - 2.1.7 (2026-02-09) - 2.1.6 (2026-02-09) - 2.1.5 (2026-02-09) - 2.1.4 (2026-02-08) - 2.1.3 (2026-02-08) - 2.1.2 (2026-02-08) - 2.1.1 (2026-02-08) - 2.1.0 (2026-02-08) - 2.0.0 (2026-02-08) - 1.0.0 (2026-02-03) - 0.12.0 (2026-01-25) - 0.11.0 (2026-01-10) - 0.10.0 (2026-01-08) - 0.9.0 (2026-01-07) - 0.8.0 (2026-01-06) - 0.7.2 (2026-01-06) - 0.7.1 (2026-01-06) - 0.7.0 (2026-01-05) - 0.6.0 (2025-12-31) - 0.5.0 (2025-12-30) - 0.4.0 (2025-12-29) - 0.3.5 (2025-12-21) - 0.3.4 (2025-12-21) - 0.3.3 (2025-12-21) - 0.3.2 (2025-12-19) - 0.3.1 (2025-12-18) - 0.3.0 (2025-12-17) - 0.2.1 (2025-12-10) - 0.2.0 (2025-12-10) - 0.1.1 (2025-12-10) - 0.1.0 (2025-12-09) +- Earlier releases: +- 8.0.4 (2026-05-28) +- 8.0.3 (2026-05-28) +- 8.0.2 (2026-05-28) +- 8.0.1 (2026-05-28) +- 8.0.0 (2026-05-27) +- 7.0.0 (2026-05-08) +- 6.0.0 (2026-04-25) +- 5.2.0 (2026-03-21) +- 5.1.0 (2026-02-28) +- 5.0.0 (2026-02-25) +- 4.1.0 (2026-02-17) +- 4.0.3 (2026-02-16) +- 4.0.2 (2026-02-15) +- 4.0.1 (2026-02-15) +- 4.0.0 (2026-02-13) +- 3.0.0 (2026-02-11) +- 2.1.9 (2026-02-10) +- 2.1.8 (2026-02-09) +- 2.1.7 (2026-02-09) +- 2.1.6 (2026-02-09) +- 2.1.5 (2026-02-09) +- 2.1.4 (2026-02-08) +- 2.1.3 (2026-02-08) +- 2.1.2 (2026-02-08) +- 2.1.1 (2026-02-08) +- 2.1.0 (2026-02-08) +- 2.0.0 (2026-02-08) +- 1.0.0 (2026-02-03) +- 0.12.0 (2026-01-25) +- 0.11.0 (2026-01-10) +- 0.10.0 (2026-01-08) +- 0.9.0 (2026-01-07) +- 0.8.0 (2026-01-06) +- 0.7.2 (2026-01-06) +- 0.7.1 (2026-01-06) +- 0.7.0 (2026-01-05) +- 0.6.0 (2025-12-31) +- 0.5.0 (2025-12-30) +- 0.4.0 (2025-12-29) +- 0.3.5 (2025-12-21) +- 0.3.4 (2025-12-21) +- 0.3.3 (2025-12-21) +- 0.3.2 (2025-12-19) +- 0.3.1 (2025-12-18) +- 0.3.0 (2025-12-17) +- 0.2.1 (2025-12-10) +- 0.2.0 (2025-12-10) +- 0.1.1 (2025-12-10) +- 0.1.0 (2025-12-09) diff --git a/plugins/README.md b/plugins/README.md index b4609df361..69025524bc 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -2,6 +2,12 @@ This document is the central reference for plugin types and plugin placement in Infinito.Nexus. +> **Not to be confused with application addons.** +> This `plugins/` directory holds **Ansible** plugins (action / filter / lookup) that extend the automation engine itself. +> Some applications *also* call their own extensions "plugins" (e.g. Nextcloud, WordPress). +> Those application-level extensions, whatever the app calls them natively, are declared through the unified `meta/addons/` contract, documented in [Unified Addons](../docs/contributing/design/role/services/layout.md#unified-addons-metaaddons-). +> An app "plugin" is an *addon*; an Ansible plugin lives here. + ## Current Project Layout ๐Ÿ—‚๏ธ ```text @@ -113,6 +119,6 @@ Use to add custom Jinja2 test operators for conditional logic. ## References ๐Ÿ”— -- Ansible plugin development: -- Ansible plugin index (all types): +- Ansible plugin development: [docs.ansible.com](https://docs.ansible.com/ansible/latest/dev_guide/developing_plugins.html) +- Ansible plugin index (all types): [docs.ansible.com](https://docs.ansible.com/ansible/latest/plugins/plugins.html) - Ansible `ansible-doc` plugin types: `ansible-doc -t ...` diff --git a/plugins/filter/node_autosize.py b/plugins/filter/node_autosize.py deleted file mode 100644 index 28db387ec3..0000000000 --- a/plugins/filter/node_autosize.py +++ /dev/null @@ -1,147 +0,0 @@ -# Reuse app config to derive sensible Node.js heap sizes for containers. -# -# Usage example (Jinja): -# {{ lookup('applications') | node_max_old_space_size('web-app-nextcloud', 'whiteboard') }} -# -# Heuristics (defaults): -# - candidate = 35% of mem_limit -# - min = 768 MB (required minimum) -# - cap = min(3072 MB, 60% of mem_limit) -# -# NEW: If mem_limit (container cgroup RAM) is smaller than min_mb, we raise an -# exception โ€” to prevent a misconfiguration where Node's heap could exceed the cgroup -# and be OOM-killed. - -from __future__ import annotations - -import re - -from ansible.errors import AnsibleFilterError - -# Import the shared config resolver from utils -try: - from utils.roles.applications.config import AppConfigKeyError, get -except Exception as e: - raise AnsibleFilterError( - f"Failed to import get from utils.roles.applications.config: {e}" - ) from e - -_SIZE_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([kmgtp]?i?b?)?\s*$", re.IGNORECASE) -_MULT = { - "": 1, - "b": 1, - "k": 10**3, - "kb": 10**3, - "m": 10**6, - "mb": 10**6, - "g": 10**9, - "gb": 10**9, - "t": 10**12, - "tb": 10**12, - "p": 10**15, - "pb": 10**15, - "kib": 1024, - "mib": 1024**2, - "gib": 1024**3, - "tib": 1024**4, - "pib": 1024**5, -} - - -def _to_bytes(val): - """Convert numeric or string memory limits (e.g. '512m', '2GiB') to bytes.""" - if val is None or val == "": - return None - if isinstance(val, (int, float)): - return int(val) - if not isinstance(val, str): - raise AnsibleFilterError(f"Unsupported mem_limit type: {type(val).__name__}") - m = _SIZE_RE.match(val) - if not m: - raise AnsibleFilterError(f"Unrecognized mem_limit string: {val!r}") - num = float(m.group(1)) - unit = (m.group(2) or "").lower() - if unit not in _MULT: - raise AnsibleFilterError(f"Unknown unit in mem_limit: {unit!r}") - return int(num * _MULT[unit]) - - -def _mb(bytes_val: int) -> int: - """Return decimal MB (10^6) as integer โ€” Node expects MB units.""" - return round(bytes_val / 10**6) - - -def _compute_old_space_mb( - total_mb: int, pct: float, min_mb: int, hardcap_mb: int, safety_cap_pct: float -) -> int: - """ - Compute Node.js old-space heap (MB) with safe minimum and cap handling. - - NOTE: The calling function ensures total_mb >= min_mb; here we only - apply the sizing heuristics and caps. - """ - candidate = int(total_mb * float(pct)) - safety_cap = int(total_mb * float(safety_cap_pct)) - final_cap = min(int(hardcap_mb), safety_cap) - - # Enforce minimum first; only apply cap if it's above the minimum - candidate = max(candidate, int(min_mb)) - if final_cap >= int(min_mb): - candidate = min(candidate, final_cap) - - # Never below a tiny hard floor - return max(candidate, 128) - - -def node_max_old_space_size( - applications: dict, - application_id: str, - service_name: str, - pct: float = 0.35, - min_mb: int = 768, - hardcap_mb: int = 3072, - safety_cap_pct: float = 0.60, -) -> int: - """ - Derive Node.js --max-old-space-size (MB) from the service's mem_limit in app config. - - Looks up: services..mem_limit for the given application_id. - - Raises: - AnsibleFilterError if mem_limit is missing/invalid OR if mem_limit (MB) < min_mb. - """ - try: - mem_limit = get( - applications=applications, - application_id=application_id, - config_path=f"services.{service_name}.mem_limit", - strict=True, - default=None, - ) - except AppConfigKeyError as e: - raise AnsibleFilterError(str(e)) from e - - if mem_limit in (None, False, ""): - raise AnsibleFilterError( - f"mem_limit not set for application '{application_id}', service '{service_name}'" - ) - - total_bytes = _to_bytes(mem_limit) - total_mb = _mb(total_bytes) - - # NEW: guardrail โ€” refuse to size a heap larger than the cgroup limit - if total_mb < int(min_mb): - raise AnsibleFilterError( - f"mem_limit ({total_mb} MB) is below the required minimum heap ({int(min_mb)} MB) " - f"for application '{application_id}', service '{service_name}'. " - f"Increase mem_limit or lower min_mb." - ) - - return _compute_old_space_mb(total_mb, pct, min_mb, hardcap_mb, safety_cap_pct) - - -class FilterModule: - def filters(self): - return { - "node_max_old_space_size": node_max_old_space_size, - } diff --git a/plugins/lookup/addon_env_flags.py b/plugins/lookup/addon_env_flags.py new file mode 100644 index 0000000000..00e211cdbd --- /dev/null +++ b/plugins/lookup/addon_env_flags.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import re +from typing import Any + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + +from utils.cache.applications import get_merged_applications +from utils.cache.base import _render_with_templar +from utils.roles.applications.config import get + +ENV_SUFFIX = "_ADDON_ENABLED" +_TRUE_TOKENS = {"true", "1", "yes", "on", "t", "y"} + + +def env_key(addon_id: str) -> str: + """Mirror addon-gating.js envKey(): upper-case, non-alphanumeric runs -> '_'.""" + return re.sub(r"[^A-Z0-9]+", "_", str(addon_id).upper()) + ENV_SUFFIX + + +def _is_enabled(value: Any) -> bool: + if isinstance(value, bool): + return value + return str(value).strip().lower() in _TRUE_TOKENS + + +def _resolve_deployed_roles(variables, templar, applications): + """Resolve TEST_E2E_PLAYWRIGHT_APPS (the deployed playwright role ids) to a set. + Returns None when it cannot be resolved, in which case bridge-deployment + gating is skipped (no behaviour change).""" + raw = (variables or {}).get("TEST_E2E_PLAYWRIGHT_APPS") + if raw is None: + return None + resolved = raw + if isinstance(raw, str): + try: + resolved = _render_with_templar( + raw, templar=templar, variables=variables, raw_applications=applications + ) + except Exception: + return None + if isinstance(resolved, str): + resolved = [r for r in re.split(r"[\s,]+", resolved.strip()) if r] + if not isinstance(resolved, (list, tuple, set)): + return None + return {str(r).strip() for r in resolved if str(r).strip()} + + +def _any_bridge_partner_deployed(bridges, deployed_roles): + """A bridged addon's partner counts as deployed iff some deployed role id + equals or ends with the bridge name (web-app-, web-svc-, ...).""" + for bridge in bridges: + name = str(bridge).strip() + if not name: + continue + variants = {name, name.replace("_", "-")} + for role in deployed_roles: + if role in variants or any(role.endswith("-" + v) for v in variants): + return True + return False + + +class LookupModule(LookupBase): + """ + lookup('addon_env_flags', application_id) + + Single source of truth for the per-addon enable flags consumed by the + Playwright addon-gating helper (``tests/addon-gating.js`` -> + ``skipUnlessAddonEnabled``). Replaces the error-prone per-template Jinja + loop: it resolves the materialised ``addons`` mapping for ``application_id`` + and returns one ``_ADDON_ENABLED=`` line per declared + addon (sorted by env key, newline-joined), where the env-key derivation + matches addon-gating.js exactly. + + A flag is ``true`` only when the addon is both ``enabled`` AND + ``required``. Optional addons (``required: false``) are intentionally + skipped by the spec suite โ€” they are a variant axis, not a guaranteed + surface โ€” so their gate flag is ``false`` even when enabled. + """ + + def run(self, terms, variables: dict[str, Any] | None = None, **kwargs): + if not terms or len(terms) != 1: + raise AnsibleError( + "lookup('addon_env_flags', application_id) expects exactly one term." + ) + + application_id = terms[0] + templar = getattr(self, "_templar", None) + variables = variables or getattr(self._templar, "available_variables", {}) or {} + + applications = get_merged_applications( + variables=variables, + roles_dir=kwargs.get("roles_dir"), + templar=templar, + ) + addons = get( + applications=applications, + application_id=application_id, + config_path="addons", + strict=False, + default={}, + skip_missing_app=True, + ) + addons = _render_with_templar( + addons, + templar=templar, + variables=variables, + raw_applications=applications, + ) + if not isinstance(addons, dict): + addons = {} + + deployed_roles = _resolve_deployed_roles(variables, templar, applications) + + lines = [] + for addon_id in sorted(addons, key=env_key): + spec = ( + addons.get(addon_id) if isinstance(addons.get(addon_id), dict) else {} + ) + active = _is_enabled(spec.get("enabled", False)) and _is_enabled( + spec.get("required", False) + ) + if active and deployed_roles is not None: + bridges = spec.get("bridges") + if isinstance(bridges, list) and bridges: + active = _any_bridge_partner_deployed(bridges, deployed_roles) + lines.append(f"{env_key(addon_id)}={'true' if active else 'false'}") + + return ["\n".join(lines)] diff --git a/plugins/lookup/api.py b/plugins/lookup/api.py new file mode 100644 index 0000000000..9740e8d16b --- /dev/null +++ b/plugins/lookup/api.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + """ + lookup('api', '.') e.g. lookup('api', 'github.client_id') + + Single point of truth for proprietary / external API credentials. Resolves a + dotted path against the global ``API`` mapping (group_vars/all/18_api.yml). + Routing every access through this plugin keeps one place to change how + proprietary API credentials are sourced (vault, env, secret manager, a + different provider abstraction) without touching the call sites. + """ + + def run(self, terms, variables: dict[str, Any] | None = None, **kwargs): + if not terms or len(terms) != 1: + raise AnsibleError( + "lookup('api') takes exactly one term: '.' " + "(e.g. 'github.client_id')." + ) + + templar = getattr(self, "_templar", None) + variables = variables or getattr(templar, "available_variables", {}) or {} + + path = str(terms[0]).strip() + if not path: + raise AnsibleError("lookup('api'): empty key path.") + + api = variables.get("API", {}) + if templar is not None: + api = templar.template(api) + if not isinstance(api, dict): + raise AnsibleError("lookup('api'): global 'API' is not a mapping.") + + node: Any = api + for part in path.split("."): + if not isinstance(node, dict) or part not in node: + raise AnsibleError( + f"lookup('api'): unknown key '{path}' (failed at '{part}'). " + f"Declare it under 'API' in group_vars/all/18_api.yml." + ) + node = node[part] + return [node] diff --git a/plugins/lookup/node_max_old_space_size.py b/plugins/lookup/node_max_old_space_size.py new file mode 100644 index 0000000000..ac8f6d4719 --- /dev/null +++ b/plugins/lookup/node_max_old_space_size.py @@ -0,0 +1,125 @@ +"""Derive a Node.js ``--max-old-space-size`` (MB) from a service's mem_limit. + +Lookup form โ€” the app config is resolved internally, so callers no longer pipe +``lookup('applications')`` in: + + {{ lookup('node_max_old_space_size', application_id, service_name) }} + {{ lookup('node_max_old_space_size', application_id, service_name, + pct=0.125, min_mb=256, hardcap_mb=1024) }} + +Heuristics (defaults): + - candidate = 35% of mem_limit + - min = 768 MB (required minimum) + - cap = min(3072 MB, 60% of mem_limit) + +If mem_limit (container cgroup RAM) is smaller than ``min_mb`` an exception is +raised, to prevent Node's heap from exceeding the cgroup and being OOM-killed. +""" + +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + +from utils.cache.applications import get_merged_applications +from utils.roles.applications.config import AppConfigKeyError, get +from utils.sizes import to_bytes + + +def _mb(bytes_val: int) -> int: + """Return decimal MB (10^6) as integer โ€” Node expects MB units.""" + return round(bytes_val / 10**6) + + +def _compute_old_space_mb( + total_mb: int, pct: float, min_mb: int, hardcap_mb: int, safety_cap_pct: float +) -> int: + """Compute Node.js old-space heap (MB) with safe minimum and cap handling. + + NOTE: The caller ensures total_mb >= min_mb; here we only apply the sizing + heuristics and caps.""" + candidate = int(total_mb * float(pct)) + safety_cap = int(total_mb * float(safety_cap_pct)) + final_cap = min(int(hardcap_mb), safety_cap) + + candidate = max(candidate, int(min_mb)) + if final_cap >= int(min_mb): + candidate = min(candidate, final_cap) + + return max(candidate, 128) + + +def node_max_old_space_size( + applications: dict, + application_id: str, + service_name: str, + pct: float = 0.35, + min_mb: int = 768, + hardcap_mb: int = 3072, + safety_cap_pct: float = 0.60, +) -> int: + """Derive Node.js --max-old-space-size (MB) from services..mem_limit. + + Raises AnsibleError if mem_limit is missing/invalid OR if mem_limit (MB) < min_mb. + """ + try: + mem_limit = get( + applications=applications, + application_id=application_id, + config_path=f"services.{service_name}.mem_limit", + strict=True, + default=None, + ) + except AppConfigKeyError as e: + raise AnsibleError(str(e)) from e + + if mem_limit in (None, False, ""): + raise AnsibleError( + f"mem_limit not set for application '{application_id}', service '{service_name}'" + ) + + total_mb = _mb(to_bytes(mem_limit)) + + if total_mb < int(min_mb): + raise AnsibleError( + f"mem_limit ({total_mb} MB) is below the required minimum heap ({int(min_mb)} MB) " + f"for application '{application_id}', service '{service_name}'. " + f"Increase mem_limit or lower min_mb." + ) + + return _compute_old_space_mb(total_mb, pct, min_mb, hardcap_mb, safety_cap_pct) + + +class LookupModule(LookupBase): + """lookup('node_max_old_space_size', application_id, service_name[, pct=, min_mb=, hardcap_mb=, safety_cap_pct=])""" + + def run(self, terms, variables: dict[str, Any] | None = None, **kwargs): + if len(terms) != 2: + raise AnsibleError( + "lookup('node_max_old_space_size', application_id, service_name" + "[, pct=, min_mb=, hardcap_mb=, safety_cap_pct=]) expects exactly " + "2 positional terms." + ) + application_id, service_name = terms[0], terms[1] + + templar = getattr(self, "_templar", None) + variables = variables or getattr(self._templar, "available_variables", {}) or {} + applications = get_merged_applications( + variables=variables, + roles_dir=kwargs.get("roles_dir"), + templar=templar, + ) + + return [ + node_max_old_space_size( + applications, + application_id, + service_name, + pct=float(kwargs.get("pct", 0.35)), + min_mb=int(kwargs.get("min_mb", 768)), + hardcap_mb=int(kwargs.get("hardcap_mb", 3072)), + safety_cap_pct=float(kwargs.get("safety_cap_pct", 0.60)), + ) + ] diff --git a/plugins/lookup/playwright_workers.py b/plugins/lookup/playwright_workers.py new file mode 100644 index 0000000000..7c2e4cf257 --- /dev/null +++ b/plugins/lookup/playwright_workers.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +from ansible.plugins.lookup import LookupBase + +# Conservative sizing for browser e2e that runs against the live, co-resident +# application stack. The binding constraint is rarely raw CPU: it is the +# application-under-test concurrency (Keycloak/Synapse rate limits, DB +# connections), RAM per Chromium worker, and the deployed stack sharing the +# host. So divide CPUs aggressively, reserve RAM for the stack, and hard-cap. +_PER_WORKER_GB = 1.5 # Chromium + node + headroom per worker +_RAM_FRACTION = 0.5 # leave half of RAM for the deployed stack under test +_CPU_DIVISOR = 4 # browser workers are heavy and the stack is co-resident +_HARD_CAP = 6 # app-under-test concurrency ceiling (login/rate limits) +_CI_CAP = 2 # CI runners: stay minimal and stable +_CI_ENV = ("CI", "GITHUB_ACTIONS", "GITLAB_CI", "BUILDKITE", "JENKINS_URL") + + +def _cpu_count() -> int: + try: + return max(1, len(os.sched_getaffinity(0))) + except (AttributeError, OSError): + return max(1, os.cpu_count() or 1) + + +def _ram_gb() -> float: + try: + with Path("/proc/meminfo").open(encoding="utf-8") as fh: + for line in fh: + if line.startswith("MemTotal:"): + return int(line.split()[1]) / (1024.0 * 1024.0) + except (OSError, ValueError, IndexError): + pass + return 4.0 # conservative fallback when /proc/meminfo is unreadable + + +def _is_ci() -> bool: + return any(os.environ.get(k) for k in _CI_ENV) + + +def compute_workers( + cpus: int, + ram_gb: float, + ci: bool, + *, + per_worker_gb: float = _PER_WORKER_GB, + ram_fraction: float = _RAM_FRACTION, + cpu_divisor: int = _CPU_DIVISOR, + hard_cap: int = _HARD_CAP, + ci_cap: int = _CI_CAP, +) -> int: + cpu_workers = max(1, cpus // max(1, cpu_divisor)) + ram_workers = max(1, int((ram_gb * ram_fraction) // per_worker_gb)) + workers = min(cpu_workers, ram_workers, hard_cap) + if ci: + workers = min(workers, ci_cap) + return max(1, workers) + + +class LookupModule(LookupBase): + """ + lookup('playwright_workers') + + Conservative, stable parallel-worker count for the Playwright e2e suite. + Derived from the control host's effective CPU and RAM, divided down because + browser workers are heavy and run against the co-resident application stack, + reserved against RAM, hard-capped for the application-under-test's + concurrency limits, and reduced further on CI. Always returns an int >= 1. + + Tunable via kwargs: cpu_divisor, hard_cap, ci_cap, per_worker_gb, + ram_fraction. + """ + + def run(self, terms, variables: dict[str, Any] | None = None, **kwargs): + return [ + compute_workers( + _cpu_count(), + _ram_gb(), + _is_ci(), + per_worker_gb=float(kwargs.get("per_worker_gb", _PER_WORKER_GB)), + ram_fraction=float(kwargs.get("ram_fraction", _RAM_FRACTION)), + cpu_divisor=int(kwargs.get("cpu_divisor", _CPU_DIVISOR)), + hard_cap=int(kwargs.get("hard_cap", _HARD_CAP)), + ci_cap=int(kwargs.get("ci_cap", _CI_CAP)), + ) + ] diff --git a/pyproject.toml b/pyproject.toml index 1865f7c5ed..db350c33e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "infinito-nexus" -version = "9.3.0" +version = "11.1.0" description = "Infinito.Nexus" readme = "README.md" requires-python = ">=3.11" diff --git a/roles/desk-chromium/README.md b/roles/desk-chromium/README.md index 62d119f23d..458573265e 100644 --- a/roles/desk-chromium/README.md +++ b/roles/desk-chromium/README.md @@ -20,6 +20,18 @@ The purpose of this role is to automate the provisioning of a secure Chromium en - **Cross-Platform Support:** Handles package variations for multiple Linux distributions. - **Seamless Integration:** Provides a stable and secure browsing setup as part of broader system automation workflows. +## Addons + +This role force-installs the following Chromium browser extensions via managed Enterprise Policies. They are declared in [meta/addons/](meta/addons/) and rendered into the CRX install list. + +| Addon | Mechanism | Default state | Bridges | +|---|---|---|---| +| ublock-origin | extension | enabled (required) | none | +| keepassxc-browser | extension | enabled (required) | none | +| privacy-badger | extension | enabled (required) | none | + +Playwright exemption: these are desktop browser extensions with no in-app web surface to drive, so no Playwright spec is required (requirement 026, Decision 11). + ## Credits Developed and maintained by **Kevin Veen-Birkenbach**. diff --git a/roles/desk-chromium/meta/addons/keepassxc-browser.yml b/roles/desk-chromium/meta/addons/keepassxc-browser.yml new file mode 100644 index 0000000000..1a5da71367 --- /dev/null +++ b/roles/desk-chromium/meta/addons/keepassxc-browser.yml @@ -0,0 +1,9 @@ +--- +required: true +mechanism: extension +source: upstream +config: + crx_id: ddkjiahejlhfcafbddmgiahcphecmpfh + update_url: "https://clients2.google.com/service/update2/crx" + installation_mode: force_installed + incognito: false diff --git a/roles/desk-chromium/meta/addons/privacy-badger.yml b/roles/desk-chromium/meta/addons/privacy-badger.yml new file mode 100644 index 0000000000..1c9bf61a55 --- /dev/null +++ b/roles/desk-chromium/meta/addons/privacy-badger.yml @@ -0,0 +1,9 @@ +--- +required: true +mechanism: extension +source: upstream +config: + crx_id: dmghijelimhndkbmpgbldicpogfkceaj + update_url: "https://clients2.google.com/service/update2/crx" + installation_mode: force_installed + incognito: true diff --git a/roles/desk-chromium/meta/addons/ublock-origin.yml b/roles/desk-chromium/meta/addons/ublock-origin.yml new file mode 100644 index 0000000000..c2d7072cf3 --- /dev/null +++ b/roles/desk-chromium/meta/addons/ublock-origin.yml @@ -0,0 +1,9 @@ +--- +required: true +mechanism: extension +source: upstream +config: + crx_id: cjpalhdlnbpafiamejdnhcphjbkeiagm + update_url: "https://clients2.google.com/service/update2/crx" + installation_mode: force_installed + incognito: true diff --git a/roles/desk-chromium/meta/services.yml b/roles/desk-chromium/meta/services.yml index 6187fa58eb..184fa3bee8 100644 --- a/roles/desk-chromium/meta/services.yml +++ b/roles/desk-chromium/meta/services.yml @@ -2,19 +2,6 @@ chromium: password_manager_enabled: false default_installation_mode: allowed - plugins: - - id: cjpalhdlnbpafiamejdnhcphjbkeiagm - update_url: https://clients2.google.com/service/update2/crx - incognito: true - installation_mode: force_installed - - id: ddkjiahejlhfcafbddmgiahcphecmpfh - update_url: https://clients2.google.com/service/update2/crx - incognito: false - installation_mode: force_installed - - id: dmghijelimhndkbmpgbldicpogfkceaj - update_url: https://clients2.google.com/service/update2/crx - incognito: true - installation_mode: force_installed lifecycle: pre-alpha min_storage: "500MB" cpus: "1.0" diff --git a/roles/desk-chromium/templates/extensions_policy.json.j2 b/roles/desk-chromium/templates/extensions_policy.json.j2 index e90f2dd615..7b536ee395 100644 --- a/roles/desk-chromium/templates/extensions_policy.json.j2 +++ b/roles/desk-chromium/templates/extensions_policy.json.j2 @@ -1,18 +1,18 @@ { "ExtensionInstallForcelist": [ - {% for plugin in lookup('config', application_id, 'services.chromium.plugins') -%} - "{{ plugin.id }};{{ plugin.update_url }}"{% if not loop.last %},{% endif %} + {% for addon in lookup('config', application_id, 'addons').values() -%} + "{{ addon.config.crx_id }};{{ addon.config.update_url }}"{% if not loop.last %},{% endif %} {% endfor %} ], "ExtensionSettings": { "*": { "installation_mode": "{{ lookup('config', application_id, 'services.chromium.default_installation_mode') }}" } - {% for plugin in lookup('config', application_id, 'services.chromium.plugins') -%}, - "{{ plugin.id }}": { - "installation_mode": "{{ plugin.installation_mode }}", - "update_url": "{{ plugin.update_url }}", - "incognito_mode": "{{ 'enabled' if plugin.incognito else 'disabled' }}" + {% for addon in lookup('config', application_id, 'addons').values() -%}, + "{{ addon.config.crx_id }}": { + "installation_mode": "{{ addon.config.installation_mode }}", + "update_url": "{{ addon.config.update_url }}", + "incognito_mode": "{{ 'enabled' if addon.config.incognito else 'disabled' }}" } {% endfor %} }, diff --git a/roles/desk-firefox/README.md b/roles/desk-firefox/README.md index 59622904bb..a73ef74c33 100644 --- a/roles/desk-firefox/README.md +++ b/roles/desk-firefox/README.md @@ -19,6 +19,20 @@ The role automates the provisioning of a secure Firefox environment, reducing ma - **Streamlined Configuration:** Automatically creates necessary directories and applies correct file permissions. - **Seamless Integration:** Easily integrates with other automation roles for a complete system setup. +## Addons + +Role-level extensions are declared in [`meta/addons/`](./meta/addons/) (unified addon contract, requirement 026): + +| Addon | Mechanism | Default state | Bridges | +|-------|-----------|---------------|---------| +| `ublock-origin` | `extension` | always installed (`required: true`) | none | +| `keepassxc-browser` | `extension` | always installed (`required: true`) | none | + +Both extensions are force-installed through Firefox Enterprise Policies; each addon's `config.xpi_url` is read by [`templates/policies.json.j2`](./templates/policies.json.j2) into `policies.Extensions.Install[]`. +They carry no cross-role dependency, so no `bridges:` key is present. + +**Playwright exemption:** these are desktop browser extensions with no in-app web surface to drive, so they are exempt from the per-addon Playwright spec (requirement 026, Decision 11). + ## Credits Developed and maintained by **Kevin Veen-Birkenbach**. diff --git a/roles/desk-firefox/meta/addons/keepassxc-browser.yml b/roles/desk-firefox/meta/addons/keepassxc-browser.yml new file mode 100644 index 0000000000..fb041e74d6 --- /dev/null +++ b/roles/desk-firefox/meta/addons/keepassxc-browser.yml @@ -0,0 +1,6 @@ +--- +required: true +mechanism: extension +source: upstream +config: + xpi_url: "https://addons.mozilla.org/firefox/downloads/latest/keepassxc-browser/latest.xpi" diff --git a/roles/desk-firefox/meta/addons/ublock-origin.yml b/roles/desk-firefox/meta/addons/ublock-origin.yml new file mode 100644 index 0000000000..ecd4f95491 --- /dev/null +++ b/roles/desk-firefox/meta/addons/ublock-origin.yml @@ -0,0 +1,6 @@ +--- +required: true +mechanism: extension +source: upstream +config: + xpi_url: "https://addons.mozilla.org/firefox/downloads/latest/ublock-origin/latest.xpi" diff --git a/roles/desk-firefox/meta/services.yml b/roles/desk-firefox/meta/services.yml index c72309ac4f..f94cb32261 100644 --- a/roles/desk-firefox/meta/services.yml +++ b/roles/desk-firefox/meta/services.yml @@ -1,8 +1,5 @@ --- firefox: - plugins: - - https://addons.mozilla.org/firefox/downloads/latest/ublock-origin/latest.xpi - - https://addons.mozilla.org/firefox/downloads/latest/keepassxc-browser/latest.xpi lifecycle: pre-alpha min_storage: "500MB" cpus: "1.0" diff --git a/roles/desk-firefox/templates/policies.json.j2 b/roles/desk-firefox/templates/policies.json.j2 index 8f08547c4a..e93cd3f653 100644 --- a/roles/desk-firefox/templates/policies.json.j2 +++ b/roles/desk-firefox/templates/policies.json.j2 @@ -2,8 +2,8 @@ "policies": { "Extensions": { "Install": [ - {% for plugin in lookup('config', application_id, 'services.firefox.plugins') -%} - "{{ plugin }}"{% if not loop.last %},{% endif %} + {% for addon in lookup('config', application_id, 'addons').values() -%} + "{{ addon.config.xpi_url }}"{% if not loop.last %},{% endif %} {% endfor %} ] }, diff --git a/roles/desk-gnome-extensions/README.md b/roles/desk-gnome-extensions/README.md index 2a4bec044e..ec7fe547e0 100644 --- a/roles/desk-gnome-extensions/README.md +++ b/roles/desk-gnome-extensions/README.md @@ -21,6 +21,14 @@ The purpose of this role is to enhance and customize the GNOME desktop environme - Executes extension configuration commands for streamlined management. - Provides an automated method for managing and updating GNOME extensions. +## Addons + +This role is a generic GNOME-extension installer driven by `services.gnome-extensions.plugins`, a 3-tuple list (`action`, `uuid`, `url`) consumed by `cli-gnome-extension-manager`. +It ships no concrete extensions of its own and reads no `meta/addons/` files. +Roles that ship concrete GNOME extensions, such as [desk-gnome](../desk-gnome/), declare them through the unified addon contract under `meta/addons/` (requirement 026), with one file per extension (`mechanism: extension`, `source: upstream`, `config: { uuid, url }`); see that role for the per-file example. + +Playwright exemption: GNOME Shell extensions are desktop-only and have no in-app web surface to drive, so they carry no Playwright spec (Confirmed Decision 11). + ## Credits Developed and maintained by **Kevin Veen-Birkenbach**. diff --git a/roles/desk-gnome/README.md b/roles/desk-gnome/README.md index 63390d7edc..bf77dae636 100644 --- a/roles/desk-gnome/README.md +++ b/roles/desk-gnome/README.md @@ -24,6 +24,19 @@ The purpose of this role is to provide a complete GNOME desktop experience by or - Installs GNOME Terminal for a robust command-line interface. - Ensures a seamless and uniform GNOME environment on Arch Linux. +## Addons + +Role-level GNOME Shell extensions are declared in [`meta/addons/`](./meta/addons/) (unified addon contract, requirement 026). +They are installed through `cli-gnome-extension-manager`, which receives the `action`, `uuid`, and `url` from each addon's `config` payload: + +| Addon | Mechanism | Default state | Bridges | +|-------|-----------|---------------|---------| +| `nasa-apod` | `extension` | required (always enabled) | none | +| `dash-to-dock` | `extension` | optional, disabled by default | none | +| `dash-to-panel` | `extension` | required (always enabled) | none | + +These are desktop GNOME Shell extensions with no in-app web surface to drive, so they are exempt from the Playwright requirement (requirement 026, Decision 11). + ## Credits Developed and maintained by **Kevin Veen-Birkenbach**. diff --git a/roles/desk-gnome/meta/addons/dash-to-dock.yml b/roles/desk-gnome/meta/addons/dash-to-dock.yml new file mode 100644 index 0000000000..cf6f5ca303 --- /dev/null +++ b/roles/desk-gnome/meta/addons/dash-to-dock.yml @@ -0,0 +1,8 @@ +--- +enabled: false +required: false +mechanism: extension +source: upstream +config: + uuid: "dash-to-dock@micxgx.gmail.com" + url: "" diff --git a/roles/desk-gnome/meta/addons/dash-to-panel.yml b/roles/desk-gnome/meta/addons/dash-to-panel.yml new file mode 100644 index 0000000000..94989afe9a --- /dev/null +++ b/roles/desk-gnome/meta/addons/dash-to-panel.yml @@ -0,0 +1,7 @@ +--- +required: true +mechanism: extension +source: upstream +config: + uuid: "dash-to-panel@jderose9.github.com" + url: "" diff --git a/roles/desk-gnome/meta/addons/nasa-apod.yml b/roles/desk-gnome/meta/addons/nasa-apod.yml new file mode 100644 index 0000000000..fbf9fce659 --- /dev/null +++ b/roles/desk-gnome/meta/addons/nasa-apod.yml @@ -0,0 +1,7 @@ +--- +required: true +mechanism: extension +source: upstream +config: + uuid: "nasa_apod@elinvention.ovh" + url: "https://github.com/Elinvention/gnome-shell-extension-nasa-apod.git" # nocheck: repository-declared diff --git a/roles/desk-gnome/meta/services.yml b/roles/desk-gnome/meta/services.yml index 5bf3b3f1c0..3b9c690282 100644 --- a/roles/desk-gnome/meta/services.yml +++ b/roles/desk-gnome/meta/services.yml @@ -1,15 +1,5 @@ --- gnome: - plugins: - - - enable - - nasa_apod@elinvention.ovh - - https://github.com/Elinvention/gnome-shell-extension-nasa-apod.git - - - disable - - dash-to-dock@micxgx.gmail.com - - "" - - - enable - - dash-to-panel@jderose9.github.com - - "" lifecycle: pre-alpha min_storage: "2GB" cpus: "1.0" diff --git a/roles/desk-gnome/meta/variants.yml b/roles/desk-gnome/meta/variants.yml new file mode 100644 index 0000000000..a327ff7a95 --- /dev/null +++ b/roles/desk-gnome/meta/variants.yml @@ -0,0 +1,10 @@ +--- +# See docs/contributing/artefact/files/role/variants.md for the file format. + +# Only the optional addon (dash-to-dock) is an axis; required addons are always installed. +- addons: + dash-to-dock: + enabled: true +- addons: + dash-to-dock: + enabled: false diff --git a/roles/desk-gnome/tasks/01_core.yml b/roles/desk-gnome/tasks/01_core.yml index dddd572ecb..0c865bc72f 100644 --- a/roles/desk-gnome/tasks/01_core.yml +++ b/roles/desk-gnome/tasks/01_core.yml @@ -42,4 +42,6 @@ loop_control: loop_var: dependency_role +- include_tasks: 02_extensions.yml + - include_tasks: utils/once/flag.yml diff --git a/roles/desk-gnome/tasks/02_extensions.yml b/roles/desk-gnome/tasks/02_extensions.yml new file mode 100644 index 0000000000..76bd2a2b62 --- /dev/null +++ b/roles/desk-gnome/tasks/02_extensions.yml @@ -0,0 +1,21 @@ +--- +- name: GNOME Activate Extensions + ansible.builtin.shell: gsettings set org.gnome.shell disable-user-extensions false + become: false + +- name: "pkgmgr install" + include_role: + name: pkgmgr-install + vars: + package_name: cli-gnome-extension-manager + +- name: Execute CLI GNOME Extension manager script + ansible.builtin.shell: >- + cli-gnome-extension-manager + "{{ 'enable' if item.value.enabled | bool else 'disable' }}" + "{{ item.value.config.uuid }}" + "{{ item.value.config.url }}" + loop: "{{ lookup('config', application_id, 'addons') | dict2items }}" + loop_control: + label: "{{ item.key }}" + become: false diff --git a/roles/svc-ai-ollama/meta/services.yml b/roles/svc-ai-ollama/meta/services.yml index d0d8237995..dc3fc8e25e 100644 --- a/roles/svc-ai-ollama/meta/services.yml +++ b/roles/svc-ai-ollama/meta/services.yml @@ -1,5 +1,6 @@ --- ollama: + bond: 1 enabled: true shared: true backup: diff --git a/roles/svc-ai-ollama/meta/variants.yml b/roles/svc-ai-ollama/meta/variants.yml index b4b78ec67f..b367c26299 100644 --- a/roles/svc-ai-ollama/meta/variants.yml +++ b/roles/svc-ai-ollama/meta/variants.yml @@ -7,11 +7,7 @@ # extra rounds. # See docs/contributing/artefact/files/role/variants.md for the file format. -# variant 0: shrink the official ollama image to alpine/ollama and -# keep the model preload list empty so GitHub Actions runners can -# complete the deploy within their disk and time budget. - - services: ollama: - preload_models: [] - image: alpine/ollama + preload_models: + - smollm2:135m diff --git a/roles/svc-db-mariadb/meta/services.yml b/roles/svc-db-mariadb/meta/services.yml index 478cda7e4d..fb3f5774fe 100644 --- a/roles/svc-db-mariadb/meta/services.yml +++ b/roles/svc-db-mariadb/meta/services.yml @@ -1,5 +1,6 @@ --- mariadb: + bond: 1 enabled: true shared: true version: "12.3" diff --git a/roles/svc-db-memcached/meta/services.yml b/roles/svc-db-memcached/meta/services.yml index 2cdbf958d7..cc5a74ae4f 100644 --- a/roles/svc-db-memcached/meta/services.yml +++ b/roles/svc-db-memcached/meta/services.yml @@ -1,5 +1,8 @@ --- memcached: + bond: 1 + enabled: true + shared: true image: memcached version: latest min_storage: 1GB diff --git a/roles/svc-db-memcached/tasks/01_core.yml b/roles/svc-db-memcached/tasks/01_core.yml new file mode 100644 index 0000000000..70042adc45 --- /dev/null +++ b/roles/svc-db-memcached/tasks/01_core.yml @@ -0,0 +1,2 @@ +--- +- include_tasks: utils/once/flag.yml diff --git a/roles/svc-db-memcached/tasks/main.yml b/roles/svc-db-memcached/tasks/main.yml index dec815dc4e..e195a12e30 100644 --- a/roles/svc-db-memcached/tasks/main.yml +++ b/roles/svc-db-memcached/tasks/main.yml @@ -1,3 +1,5 @@ --- -# Template-only role: ships the Memcached Docker Compose snippet under templates/ -# for consumer roles to render into their own compose stack. No runtime tasks. +- include_tasks: 01_core.yml + when: run_once_svc_db_memcached is not defined + vars: + application_id: svc-db-memcached diff --git a/roles/svc-db-openldap/meta/services.yml b/roles/svc-db-openldap/meta/services.yml index c758f87a09..3fcf62ece2 100644 --- a/roles/svc-db-openldap/meta/services.yml +++ b/roles/svc-db-openldap/meta/services.yml @@ -1,5 +1,6 @@ --- openldap: + bond: 1 enabled: true shared: true provides: ldap diff --git a/roles/svc-db-postgres/meta/services.yml b/roles/svc-db-postgres/meta/services.yml index 20980d1734..b2f5742ac1 100644 --- a/roles/svc-db-postgres/meta/services.yml +++ b/roles/svc-db-postgres/meta/services.yml @@ -1,5 +1,6 @@ --- postgres: + bond: 1 enabled: true shared: true image: postgis/postgis @@ -9,7 +10,7 @@ postgres: backup: database_routine: true cpus: "2.0" - mem_reservation: 4g + mem_reservation: 3g mem_limit: 6g pids_limit: 1024 ports: diff --git a/roles/svc-db-postgres/tasks/main.yml b/roles/svc-db-postgres/tasks/main.yml index 59441409b5..c8457b6b5a 100644 --- a/roles/svc-db-postgres/tasks/main.yml +++ b/roles/svc-db-postgres/tasks/main.yml @@ -12,3 +12,5 @@ - name: "Initialize dedicated database" include_tasks: 02_init.yml when: database_init | bool + vars: + application_id: svc-db-postgres diff --git a/roles/svc-db-redis/meta/services.yml b/roles/svc-db-redis/meta/services.yml index c6b6b5eda8..8d91cece0c 100644 --- a/roles/svc-db-redis/meta/services.yml +++ b/roles/svc-db-redis/meta/services.yml @@ -1,5 +1,8 @@ --- redis: + bond: 1 + enabled: true + shared: true image: redis version: alpine min_storage: 1GB diff --git a/roles/svc-db-redis/tasks/01_core.yml b/roles/svc-db-redis/tasks/01_core.yml new file mode 100644 index 0000000000..70042adc45 --- /dev/null +++ b/roles/svc-db-redis/tasks/01_core.yml @@ -0,0 +1,2 @@ +--- +- include_tasks: utils/once/flag.yml diff --git a/roles/svc-db-redis/tasks/main.yml b/roles/svc-db-redis/tasks/main.yml index a5073081bb..108a1b6b56 100644 --- a/roles/svc-db-redis/tasks/main.yml +++ b/roles/svc-db-redis/tasks/main.yml @@ -1,3 +1,5 @@ --- -# Template-only role: ships the Redis Docker Compose snippet under templates/ -# for consumer roles to render into their own compose stack. No runtime tasks. +- include_tasks: 01_core.yml + when: run_once_svc_db_redis is not defined + vars: + application_id: svc-db-redis diff --git a/roles/sys-dns-cloudflare-records/defaults/main.yml b/roles/sys-dns-cloudflare-records/defaults/main.yml index 9579fdfac4..55bc5321b9 100644 --- a/roles/sys-dns-cloudflare-records/defaults/main.yml +++ b/roles/sys-dns-cloudflare-records/defaults/main.yml @@ -1,8 +1,4 @@ --- -# Cloudflare API Token -# More information here: group_vars/all/docs/CLOUDFLARE_API_TOKEN.md -CLOUDFLARE_API_TOKEN: "" - cloudflare_async_enabled: "{{ ASYNC_ENABLED | bool }}" cloudflare_async_time: "{{ ASYNC_TIME }}" cloudflare_async_poll: "{{ ASYNC_POLL }}" diff --git a/roles/sys-dns-cloudflare-records/docs/CLOUDFLARE_API_TOKEN.md b/roles/sys-dns-cloudflare-records/docs/CLOUDFLARE_API_TOKEN.md index 4f790e257e..80aadfa6fa 100644 --- a/roles/sys-dns-cloudflare-records/docs/CLOUDFLARE_API_TOKEN.md +++ b/roles/sys-dns-cloudflare-records/docs/CLOUDFLARE_API_TOKEN.md @@ -1,10 +1,10 @@ -# Cloudflare API Token for Ansible (`CLOUDFLARE_API_TOKEN`) +# Cloudflare API Token for Ansible (`API.cloudflare.api_token`) This document explains how to generate and use a Cloudflare API Token for DNS automation and certificate operations in Ansible (e.g., with Certbot). ## Purpose -The `CLOUDFLARE_API_TOKEN` variable must contain a valid Cloudflare API Token. +The `API.cloudflare.api_token` value (declared in `group_vars/all/18_api.yml`, read via `lookup('api', 'cloudflare.api_token')`) must contain a valid Cloudflare API Token. This token is used for all DNS operations and ACME (SSL/TLS certificate) challenges that require access to your Cloudflare-managed domains. **Never commit your API token to a public repository. Always keep it secure!** @@ -58,4 +58,6 @@ Add the following permissions: Set the token in your Ansible inventory or secrets file: ```yaml -CLOUDFLARE_API_TOKEN: "cf_your_generated_token_here" +API: + cloudflare: + api_token: "cf_your_generated_token_here" diff --git a/roles/sys-dns-cloudflare-records/tasks/main.yml b/roles/sys-dns-cloudflare-records/tasks/main.yml index d737c94970..b84f34930f 100644 --- a/roles/sys-dns-cloudflare-records/tasks/main.yml +++ b/roles/sys-dns-cloudflare-records/tasks/main.yml @@ -3,13 +3,13 @@ - name: Assert token ansible.builtin.assert: - that: ["CLOUDFLARE_API_TOKEN | length > 0"] + that: ["lookup('api', 'cloudflare.api_token') | length > 0"] no_log: "{{ cloudflare_no_log | bool }}" when: MODE_ASSERT | bool - name: Apply A/AAAA community.general.cloudflare_dns: - api_token: "{{ CLOUDFLARE_API_TOKEN }}" + api_token: "{{ lookup('api', 'cloudflare.api_token') }}" zone: "{{ item.zone }}" type: "{{ item.type }}" name: "{{ item.name }}" @@ -41,7 +41,7 @@ - name: Apply CNAME/MX/TXT community.general.cloudflare_dns: - api_token: "{{ CLOUDFLARE_API_TOKEN }}" + api_token: "{{ lookup('api', 'cloudflare.api_token') }}" zone: "{{ item.zone }}" type: "{{ item.type }}" name: "{{ item.name }}" @@ -73,7 +73,7 @@ - name: Apply SRV community.general.cloudflare_dns: - api_token: "{{ CLOUDFLARE_API_TOKEN }}" + api_token: "{{ lookup('api', 'cloudflare.api_token') }}" zone: "{{ item.zone }}" type: SRV service: "{{ item.service }}" diff --git a/roles/sys-front-tls/templates/nginx_ssl_header.j2 b/roles/sys-front-tls/templates/nginx_ssl_header.j2 index ef44349a46..4ed2b5e9b2 100644 --- a/roles/sys-front-tls/templates/nginx_ssl_header.j2 +++ b/roles/sys-front-tls/templates/nginx_ssl_header.j2 @@ -11,7 +11,7 @@ ssl_session_timeout 1d; ssl_session_cache shared:SSL:50m; ssl_session_tickets on; -add_header Strict-Transport-Security max-age=15768000; +add_header Strict-Transport-Security max-age=15768000 always; ssl_stapling on; ssl_stapling_verify on; diff --git a/roles/sys-stk-front-base/tasks/01_cloudflare.yml b/roles/sys-stk-front-base/tasks/01_cloudflare.yml index a29d40a98a..efc554db69 100644 --- a/roles/sys-stk-front-base/tasks/01_cloudflare.yml +++ b/roles/sys-stk-front-base/tasks/01_cloudflare.yml @@ -17,7 +17,7 @@ url: "{{ cf_api_url }}?name={{ domain | to_primary_domain }}" method: GET headers: - Authorization: "Bearer {{ CLOUDFLARE_API_TOKEN }}" + Authorization: "Bearer {{ lookup('api', 'cloudflare.api_token') }}" Content-Type: "application/json" return_content: true register: cf_zone_lookup_dev diff --git a/roles/sys-stk-front-base/tasks/cloudflare/01_cleanup.yml b/roles/sys-stk-front-base/tasks/cloudflare/01_cleanup.yml index 2214441e66..5b8753089b 100644 --- a/roles/sys-stk-front-base/tasks/cloudflare/01_cleanup.yml +++ b/roles/sys-stk-front-base/tasks/cloudflare/01_cleanup.yml @@ -4,7 +4,7 @@ url: "https://api.cloudflare.com/client/v4/zones/{{ cf_zone_id }}/purge_cache" method: POST headers: - Authorization: "Bearer {{ CLOUDFLARE_API_TOKEN }}" + Authorization: "Bearer {{ lookup('api', 'cloudflare.api_token') }}" Content-Type: "application/json" body: purge_everything: true diff --git a/roles/sys-stk-front-base/tasks/cloudflare/02_enable_cf_dev_mode.yml b/roles/sys-stk-front-base/tasks/cloudflare/02_enable_cf_dev_mode.yml index 6929590479..1dbb37561f 100644 --- a/roles/sys-stk-front-base/tasks/cloudflare/02_enable_cf_dev_mode.yml +++ b/roles/sys-stk-front-base/tasks/cloudflare/02_enable_cf_dev_mode.yml @@ -1,6 +1,6 @@ --- # Enables Cloudflare Development Mode (bypasses cache for ~3 hours). -# Uses the same auth token as in 01_cleanup.yml: CLOUDFLARE_API_TOKEN +# Uses the same auth token as in 01_cleanup.yml: lookup('api', 'cloudflare.api_token') # Assumes `domain` and (optionally) `cf_zone_id` are available. # Safe to run repeatedly; only changes when the mode is not already "on". @@ -9,7 +9,7 @@ url: "https://api.cloudflare.com/client/v4/zones/{{ cf_zone_id }}/settings/development_mode" method: GET headers: - Authorization: "Bearer {{ CLOUDFLARE_API_TOKEN }}" + Authorization: "Bearer {{ lookup('api', 'cloudflare.api_token') }}" Content-Type: "application/json" return_content: true register: cf_dev_mode_current @@ -20,7 +20,7 @@ url: "https://api.cloudflare.com/client/v4/zones/{{ cf_zone_id }}/settings/development_mode" method: PATCH headers: - Authorization: "Bearer {{ CLOUDFLARE_API_TOKEN }}" + Authorization: "Bearer {{ lookup('api', 'cloudflare.api_token') }}" Content-Type: "application/json" body: value: "on" diff --git a/roles/sys-stk-front-proxy/tasks/main.yml b/roles/sys-stk-front-proxy/tasks/main.yml index 07089d3f66..0a59564e11 100644 --- a/roles/sys-stk-front-proxy/tasks/main.yml +++ b/roles/sys-stk-front-proxy/tasks/main.yml @@ -58,6 +58,8 @@ ) > 0 else "roles/sys-svc-proxy/templates/vhost/%s.conf.j2" | format(webserver_vhost_flavour) }} + vars: + webserver_vhost_flavour: basic - name: "Deploy NGINX config: '{{ front_proxy_domain }}' -> '{{ front_proxy_domain_conf_dst_resolved }}'" ansible.builtin.template: @@ -67,6 +69,9 @@ application_id: "{{ front_proxy_application_id }}" domain: "{{ front_proxy_domain }}" http_port: "{{ front_proxy_http_port }}" + webserver_extra_configuration: "" + webserver_websocket_location: "" + webserver_client_max_body_size: "{{ lookup('config', front_proxy_application_id, 'server.client_max_body_size', '100m') }}" register: nginx_conf notify: restart openresty diff --git a/roles/sys-svc-certbot/tasks/02_no_webroot.yml b/roles/sys-svc-certbot/tasks/02_no_webroot.yml index 1de2533865..64ffad0dca 100644 --- a/roles/sys-svc-certbot/tasks/02_no_webroot.yml +++ b/roles/sys-svc-certbot/tasks/02_no_webroot.yml @@ -16,7 +16,7 @@ copy: dest: "{{ CERTBOT_CREDENTIALS_FILE }}" content: | - dns_{{ CERTBOT_ACME_CHALLENGE_METHOD }}_api_token = {{ CLOUDFLARE_API_TOKEN }} + dns_{{ CERTBOT_ACME_CHALLENGE_METHOD }}_api_token = {{ lookup('api', 'cloudflare.api_token') }} owner: root group: root mode: "0600" diff --git a/roles/sys-svc-compose-ca/files/compose_ca.py b/roles/sys-svc-compose-ca/files/compose_ca.py index 66e0cf080b..bc33578afb 100755 --- a/roles/sys-svc-compose-ca/files/compose_ca.py +++ b/roles/sys-svc-compose-ca/files/compose_ca.py @@ -10,8 +10,10 @@ import json import os import re import shlex +import signal import subprocess import sys +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Any @@ -29,16 +31,39 @@ def die(msg: str, code: int = 2) -> None: raise SystemExit(code) -def run(cmd: list[str], *, cwd: Path, env: dict[str, str]) -> tuple[int, str, str]: - p = subprocess.run( +def run( + cmd: list[str], + *, + cwd: Path, + env: dict[str, str], + timeout: int = 600, + capture: bool = True, +) -> tuple[int, str, str]: + # killpg the whole group on timeout: a daemon-side docker/buildx grandchild + # can keep the captured pipe open and make subprocess's own timeout inert. + pipe = subprocess.PIPE if capture else subprocess.DEVNULL + proc = subprocess.Popen( cmd, cwd=str(cwd), env=env, text=True, - capture_output=True, - check=False, + stdout=pipe, + stderr=pipe, + start_new_session=True, ) - return p.returncode, (p.stdout or ""), (p.stderr or "") + try: + out, err = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except (ProcessLookupError, PermissionError): + proc.kill() + try: + out, err = proc.communicate(timeout=30) + except subprocess.TimeoutExpired: + out, err = "", "" + return 124, (out or ""), f"timed out after {timeout}s: {' '.join(cmd)}" + return proc.returncode, (out or ""), (err or "") def run_checked(cmd: list[str], *, cwd: Path, env: dict[str, str], label: str) -> None: @@ -245,22 +270,67 @@ def docker_image_exists(image: str, *, cwd: Path, env: dict[str, str]) -> bool: return rc == 0 -def docker_image_has_bin_sh(image: str, *, cwd: Path, env: dict[str, str]) -> bool: - """ - Best-effort detection whether the image provides /bin/sh. +TIMEOUT_RC = 124 - We need /bin/sh to implement a runtime wrapper entrypoint that can: - - run the CA wrapper when executable - - otherwise print a warning and continue - Distroless images usually do not have /bin/sh. - """ +def docker_image_has_bin_sh( + image: str, *, cwd: Path, env: dict[str, str] +) -> bool | None: + """Best-effort detection whether the image provides /bin/sh.""" rc, _out, _err = run( ["docker", "run", "--rm", "--entrypoint", "/bin/sh", image, "-c", "exit 0"], cwd=cwd, env=env, + timeout=60, + capture=False, ) - return rc == 0 + if rc == 0: + return True + if rc == TIMEOUT_RC: + return None + return False + + +ImageMeta = tuple[bool, list[str], list[str], bool | None] + + +def _gather_one_image(image: str, *, cwd: Path, env: dict[str, str]) -> ImageMeta: + rc, out, _err = run( + ["docker", "image", "inspect", image], cwd=cwd, env=env, timeout=90 + ) + if rc != 0: + return (False, [], [], None if rc == TIMEOUT_RC else False) + try: + data = json.loads(out) + except json.JSONDecodeError: + return (False, [], [], False) + cfg = data[0].get("Config") if isinstance(data, list) and data else None + cfg = cfg if isinstance(cfg, dict) else {} + ep = cfg.get("Entrypoint") + cmd = cfg.get("Cmd") + ep_list = ep if isinstance(ep, list) and all(isinstance(x, str) for x in ep) else [] + cmd_list = ( + cmd if isinstance(cmd, list) and all(isinstance(x, str) for x in cmd) else [] + ) + has_sh = docker_image_has_bin_sh(image, cwd=cwd, env=env) + return (True, ep_list, cmd_list, has_sh) + + +def gather_image_meta( + images: list[str], *, cwd: Path, env: dict[str, str] +) -> dict[str, ImageMeta]: + """Gather (exists, entrypoint, cmd, has_sh) per unique image, concurrently + โ€” the per-image docker reads dominate the inject and must not run serially + under DiD latency or they blow the handler timeout for many-service apps.""" + if not images: + return {} + workers = min(8, len(images)) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = { + pool.submit(_gather_one_image, image, cwd=cwd, env=env): image + for image in images + } + return {futures[f]: f.result() for f in as_completed(futures)} def _has_build(svc: dict[str, Any]) -> bool: @@ -469,7 +539,20 @@ def render_override( wrapper_container = "/tmp/infinito/bin/with-ca-trust.sh" # noqa: S108 out_services: dict[str, Any] = {} - sh_cache: dict[str, bool] = {} + + image_meta = gather_image_meta( + sorted( + { + svc["image"].strip() + for svc in services.values() + if isinstance(svc, dict) + and isinstance(svc.get("image"), str) + and svc["image"].strip() + } + ), + cwd=cwd, + env=env, + ) for name, svc in services.items(): if not isinstance(svc, dict): @@ -487,25 +570,15 @@ def render_override( ) img_ep, img_cmd = [], [] img_name = "" + has_sh = False else: img_name = image.strip() - compose_cmd = service_to_compose_cmd.get(name) - if not compose_cmd: - die(f"Internal error: missing compose cmd mapping for service '{name}'") - - ensure_image_available( - service_name=name, - svc=svc, - image=img_name, - services=services, - service_to_compose_cmd=service_to_compose_cmd, - compose_base_cmd=compose_cmd, - cwd=cwd, - env=env, + _exists, raw_ep, raw_cmd, sh_state = image_meta.get( + img_name, (False, [], [], None) ) - img_ep, img_cmd = docker_image_inspect(img_name, cwd=cwd, env=env) - img_ep = normalize_entrypoint(img_ep) - img_cmd = normalize_cmd(img_cmd) + has_sh = sh_state is not False + img_ep = normalize_entrypoint(raw_ep) + img_cmd = normalize_cmd(raw_cmd) final_ep = svc_ep or img_ep final_cmd = svc_cmd or img_cmd @@ -515,11 +588,6 @@ def render_override( effective_cmd = final_ep + final_cmd - if not effective_cmd: - die( - f"Service '{name}' resolved to empty effective command (image='{img_name or image}')" - ) - # Always inject env vars + mounts. override_svc: dict[str, Any] = { "volumes": [ @@ -536,18 +604,8 @@ def render_override( }, } - # Only override entrypoint/command when /bin/sh exists (otherwise distroless breaks). - has_sh = False - if img_name: - if img_name in sh_cache: - has_sh = sh_cache[img_name] - else: - has_sh = docker_image_has_bin_sh(img_name, cwd=cwd, env=env) - sh_cache[img_name] = has_sh - - if has_sh: + if has_sh and effective_cmd: override_svc["entrypoint"] = [wrapper_container] - # Escape $ -> $$ to prevent host-side Compose interpolation. override_svc["command"] = escape_compose_vars(effective_cmd) out_services[name] = override_svc diff --git a/roles/sys-svc-compose/handlers/main.yml b/roles/sys-svc-compose/handlers/main.yml index a9e4d06ab8..8d9ca62da8 100644 --- a/roles/sys-svc-compose/handlers/main.yml +++ b/roles/sys-svc-compose/handlers/main.yml @@ -7,6 +7,7 @@ chdir: "{{ lookup('container', application_id, 'directories.instance') }}" executable: /bin/bash register: compose_ca_override + timeout: 1200 retries: 7 delay: 20 until: compose_ca_override is succeeded diff --git a/roles/sys-svc-dns/README.md b/roles/sys-svc-dns/README.md index e8583a137a..38750cdcfd 100644 --- a/roles/sys-svc-dns/README.md +++ b/roles/sys-svc-dns/README.md @@ -24,7 +24,7 @@ This role cloudflare DNS bootstrap: parent host A/AAAA (and optional CAA), runs ## What it does -1. **Validate `CLOUDFLARE_API_TOKEN`** is present (early fail if missing). +1. **Validate the Cloudflare API token** (`API.cloudflare.api_token`) is present (early fail if missing). 2. **Ensure apex A/AAAA exist** for every **base SLD** in `SYS_SVC_DNS_BASE_DOMAINS`: - Writes `@ A` โ†’ `networks.internet.ip4` - Writes `@ AAAA` โ†’ `networks.internet.ip6` (only if global and present) diff --git a/roles/sys-svc-dns/tasks/01_core.yml b/roles/sys-svc-dns/tasks/01_core.yml index c89bcf67a7..9e70d55dc0 100644 --- a/roles/sys-svc-dns/tasks/01_core.yml +++ b/roles/sys-svc-dns/tasks/01_core.yml @@ -1,9 +1,9 @@ --- -- name: "Validate CLOUDFLARE_API_TOKEN" +- name: "Validate the Cloudflare API token" fail: msg: > - The variable "CLOUDFLARE_API_TOKEN" must be defined and cannot be empty! - when: (CLOUDFLARE_API_TOKENโ€ฏ|โ€ฏdefault('')โ€ฏ|โ€ฏtrim) == '' + API.cloudflare.api_token must be set (via lookup('api', 'cloudflare.api_token')) and cannot be empty! + when: (lookup('api', 'cloudflare.api_token') | trim) == '' - name: "Apply apex A/AAAA for base domains" include_tasks: 02_apex.yml @@ -14,7 +14,7 @@ - name: "Ensure all CAA records are present" community.general.cloudflare_dns: - api_token: "{{ CLOUDFLARE_API_TOKEN }}" + api_token: "{{ lookup('api', 'cloudflare.api_token') }}" zone: "{{ item.0 }}" record: "@" type: CAA diff --git a/roles/sys-svc-dns/tasks/02_apex.yml b/roles/sys-svc-dns/tasks/02_apex.yml index 44c4c95feb..1ad82d6815 100644 --- a/roles/sys-svc-dns/tasks/02_apex.yml +++ b/roles/sys-svc-dns/tasks/02_apex.yml @@ -1,7 +1,7 @@ --- - name: "Ensure A @ for {{ base_domain }}" community.general.cloudflare_dns: - api_token: "{{ CLOUDFLARE_API_TOKEN }}" + api_token: "{{ lookup('api', 'cloudflare.api_token') }}" zone: "{{ base_domain }}" type: A name: "@" @@ -14,7 +14,7 @@ - name: "Ensure AAAA @ for {{ base_domain }} with '{{ networks.internet.ip6 | default('None') }}" community.general.cloudflare_dns: - api_token: "{{ CLOUDFLARE_API_TOKEN }}" + api_token: "{{ lookup('api', 'cloudflare.api_token') }}" zone: "{{ base_domain }}" type: AAAA name: "@" diff --git a/roles/sys-svc-objstore/tasks/main.yml b/roles/sys-svc-objstore/tasks/main.yml index 079f9ea692..78083fd961 100644 --- a/roles/sys-svc-objstore/tasks/main.yml +++ b/roles/sys-svc-objstore/tasks/main.yml @@ -19,6 +19,10 @@ vars: application_id: "{{ _provider_role }}" domain: "{{ lookup('domain', _provider_role) }}" + database_consumer_id: "{{ _provider_role }}" + compose_repository_url: "{{ '' }}" + compose_repository_ref: "{{ '' }}" + compose_repository_pull: false when: >- lookup( 'vars', diff --git a/roles/sys-svc-proxy/templates/location/html.conf.j2 b/roles/sys-svc-proxy/templates/location/html.conf.j2 index 31ce35d4ff..48174eca43 100644 --- a/roles/sys-svc-proxy/templates/location/html.conf.j2 +++ b/roles/sys-svc-proxy/templates/location/html.conf.j2 @@ -16,7 +16,7 @@ location {{location}} {% endif %} # Client Limits for HTML - client_max_body_size {{ webserver_client_max_body_size | default('100m') }}; + client_max_body_size {{ webserver_client_max_body_size | default(lookup('config', application_id, 'server.client_max_body_size', '100m')) }}; {% set _loc = location|trim %} proxy_pass http://127.0.0.1:{{ http_port }}{{ (_loc|regex_replace('^(?:=|\\^~)\\s*','')) if not (_loc is match('^(@|~)')) else '' }}; diff --git a/roles/sys-svc-proxy/templates/location/upload.conf.j2 b/roles/sys-svc-proxy/templates/location/upload.conf.j2 index 38b9a1d73e..9504bfa700 100644 --- a/roles/sys-svc-proxy/templates/location/upload.conf.j2 +++ b/roles/sys-svc-proxy/templates/location/upload.conf.j2 @@ -13,7 +13,7 @@ location {{ location_upload }} { proxy_send_timeout 120s; # Client Limits for Upload - client_max_body_size {{ webserver_client_max_body_size }}; + client_max_body_size {{ webserver_client_max_body_size | default(lookup('config', application_id, 'server.client_max_body_size', '100m')) }}; {% include 'roles/sys-svc-proxy/templates/headers/upstream_strips.conf.j2' %} } \ No newline at end of file diff --git a/roles/sys-svc-proxy/templates/vhost/ws_generic.conf.j2 b/roles/sys-svc-proxy/templates/vhost/ws_generic.conf.j2 index 0abf7e0c78..1a4ec708f9 100644 --- a/roles/sys-svc-proxy/templates/vhost/ws_generic.conf.j2 +++ b/roles/sys-svc-proxy/templates/vhost/ws_generic.conf.j2 @@ -5,7 +5,7 @@ server { {% include 'roles/sys-front-inj-all/templates/server.conf.j2' %} - client_max_body_size {{ webserver_client_max_body_size | default('100m') }}; + client_max_body_size {{ webserver_client_max_body_size | default(lookup('config', application_id, 'server.client_max_body_size', '100m')) }}; keepalive_timeout 70; sendfile on; diff --git a/roles/test-e2e-playwright/README.md b/roles/test-e2e-playwright/README.md index ad85eb1da3..00361b9b69 100644 --- a/roles/test-e2e-playwright/README.md +++ b/roles/test-e2e-playwright/README.md @@ -23,7 +23,7 @@ Playwright inside a Docker image derived from the central Playwright package ver This role: - Discovers Playwright-enabled roles by scanning `roles/*/templates/playwright.env.j2` -- Supports allow-/deny-lists via `TEST_E2E_PLAYWRIGHT_ONLY_ROLES` and `TEST_E2E_PLAYWRIGHT_SKIP_ROLES` +- Supports allow-/deny-lists via `test_e2e_playwright_only_roles` and `test_e2e_playwright_skip_roles` - Stages each Playwright project into `TEST_E2E_PLAYWRIGHT_STAGE_BASE_DIR/` - Renders the central `package.json` template into each staged project and injects the central `playwright.config.js` - Copies every role-specific `files/playwright/*.js` (the `playwright.spec.js` aggregator plus its `test-*.js` scenario modules) into the staged `tests/` directory, and stages `files/playwright/fixtures/` (binary assets such as sample images/videos) alongside them @@ -87,14 +87,14 @@ Both are used as central defaults for every app role. ### Readiness wait -- `TEST_E2E_PLAYWRIGHT_WAIT_ENABLED` (default: `true`) -- `TEST_E2E_PLAYWRIGHT_WAIT_RETRIES` (default: `30`) -- `TEST_E2E_PLAYWRIGHT_WAIT_DELAY` (default: `5`) +- `test_e2e_playwright_wait_enabled` (default: `true`) +- `test_e2e_playwright_wait_retries` (default: `30`) +- `test_e2e_playwright_wait_delay` (default: `5`) ### Discovery filters -- `TEST_E2E_PLAYWRIGHT_ONLY_ROLES` (default: `allowed_applications`) -- `TEST_E2E_PLAYWRIGHT_SKIP_ROLES` (default: `[]`) +- `test_e2e_playwright_only_roles` (default: `allowed_applications`) +- `test_e2e_playwright_skip_roles` (default: `[]`) ## Design notes diff --git a/roles/test-e2e-playwright/defaults/main.yml b/roles/test-e2e-playwright/defaults/main.yml index 639cf28529..bca9933215 100644 --- a/roles/test-e2e-playwright/defaults/main.yml +++ b/roles/test-e2e-playwright/defaults/main.yml @@ -1,26 +1,27 @@ --- -# Where we stage per-role Playwright projects on the host -TEST_E2E_PLAYWRIGHT_STAGE_BASE_DIR: "/tmp/test-e2e-playwright" +# Hard cap per role; Ansible kills the sidecar if exceeded. Sized for +# addon-heavy roles that run many per-addon specs, e.g. nextcloud (~50 addons). +test_e2e_playwright_run_timeout: 3600 -# Where to store per-role reports/artifacts (host dir) -TEST_E2E_PLAYWRIGHT_REPORTS_BASE_DIR: "/var/lib/infinito/logs/test-e2e-playwright" +# Per-test timeout (ms) inside Playwright (config.timeout); caps a single spec. +test_e2e_playwright_test_timeout: 300000 -# `--max-failures=5` bounds the retry budget when a role is broken. -TEST_E2E_PLAYWRIGHT_COMMAND: "npx --no-install playwright test --max-failures=5" - -# Hard cap per role; Ansible kills the sidecar if exceeded. -TEST_E2E_PLAYWRIGHT_RUN_TIMEOUT: 1200 +# Parallel Playwright workers. Spec FILES run concurrently (tests within a file +# stay serial while TEST_E2E_PLAYWRIGHT_FULLY_PARALLEL is false). The count is +# computed conservatively from the host CPU/RAM, capped for the app-under-test's +# concurrency limits, and reduced on CI. +test_e2e_playwright_workers: "{{ lookup('playwright_workers') }}" # Optional readiness wait -TEST_E2E_PLAYWRIGHT_WAIT_ENABLED: true -TEST_E2E_PLAYWRIGHT_WAIT_RETRIES: 30 -TEST_E2E_PLAYWRIGHT_WAIT_DELAY: 5 +test_e2e_playwright_wait_enabled: true +test_e2e_playwright_wait_retries: 30 +test_e2e_playwright_wait_delay: 5 # Run only a subset (optional) -TEST_E2E_PLAYWRIGHT_ONLY_ROLES: "{{ lookup('deployment').groups }}" -TEST_E2E_PLAYWRIGHT_SKIP_ROLES: [] # e.g. ["svc-something"] +test_e2e_playwright_only_roles: "{{ lookup('deployment').groups }}" +test_e2e_playwright_skip_roles: [] # e.g. ["svc-something"] -TEST_E2E_PLAYWRIGHT_KEEP: false +test_e2e_playwright_keep: false # 0-based matrix-deploy round id. Pushed as an Ansible extra var by # `cli/administration/deploy/development/deploy.py` when the matrix diff --git a/roles/test-e2e-playwright/files/addon-gating.js b/roles/test-e2e-playwright/files/addon-gating.js new file mode 100644 index 0000000000..0fa74b78fb --- /dev/null +++ b/roles/test-e2e-playwright/files/addon-gating.js @@ -0,0 +1,27 @@ +const { test } = require("@playwright/test"); + +const SUFFIX = "_ADDON_ENABLED"; + +function envKey(id) { + return id.toUpperCase().replace(/[^A-Z0-9]+/g, "_") + SUFFIX; +} + +function isAddonEnabled(id) { + const raw = process.env[envKey(id)]; + if (raw === undefined) return false; + return String(raw).toLowerCase() === "true"; +} + +function addonDisabledReason(id) { + if (isAddonEnabled(id)) return null; + return `${envKey(id)} is not "true" (addon "${id}" disabled or not deployed)`; +} + +function skipUnlessAddonEnabled(id) { + const reason = addonDisabledReason(id); + if (reason !== null) { + test.skip(true, reason); + } +} + +module.exports = { envKey, isAddonEnabled, addonDisabledReason, skipUnlessAddonEnabled }; diff --git a/roles/test-e2e-playwright/files/personas/utils/logout.js b/roles/test-e2e-playwright/files/personas/utils/logout.js index 05454efc67..87ef1cdf90 100644 --- a/roles/test-e2e-playwright/files/personas/utils/logout.js +++ b/roles/test-e2e-playwright/files/personas/utils/logout.js @@ -128,7 +128,7 @@ async function inAppLogout(page) { tried.add(key); await trigger.click({ timeout: 5_000 }).catch(() => {}); // Give the dropdown / popover time to render its items. - await page.waitForTimeout(750); + await page.waitForTimeout(1_500); if (await tryLogoutFrom(page)) { await page.waitForLoadState("domcontentloaded", { timeout: 30_000 }).catch(() => {}); return; diff --git a/roles/test-e2e-playwright/files/playwright.config.js b/roles/test-e2e-playwright/files/playwright.config.js index 3ac6f2f235..d626fb6881 100644 --- a/roles/test-e2e-playwright/files/playwright.config.js +++ b/roles/test-e2e-playwright/files/playwright.config.js @@ -6,9 +6,11 @@ const keepAll = (process.env.INFINITO_PLAYWRIGHT_KEEP || "").toLowerCase() === " module.exports = defineConfig({ testDir: "./tests", - timeout: 300_000, + testMatch: "**/*.@(spec|test).js", + timeout: Number(process.env.PLAYWRIGHT_TEST_TIMEOUT) || 300_000, retries: 2, - workers: 1, + workers: Number(process.env.PLAYWRIGHT_WORKERS) || 1, + fullyParallel: (process.env.PLAYWRIGHT_FULLY_PARALLEL || "").toLowerCase() === "true", outputDir: "/reports/test-results", reporter: [ ["list"], diff --git a/roles/test-e2e-playwright/tasks/02_run_one.yml b/roles/test-e2e-playwright/tasks/02_run_one.yml index c38226d197..ae9de184db 100644 --- a/roles/test-e2e-playwright/tasks/02_run_one.yml +++ b/roles/test-e2e-playwright/tasks/02_run_one.yml @@ -148,13 +148,12 @@ dest: "{{ _playwright_stage_dir }}/playwright.config.js" mode: "0644" -- name: Copy role Playwright files (spec + companion modules) into staging tests directory +- name: Copy role Playwright tree (spec + companions + addons/ + fixtures/) into staging tests directory ansible.builtin.copy: - src: "{{ item }}" - dest: "{{ _playwright_stage_tests_dir }}/{{ item | basename }}" + src: "{{ _playwright_files_dir }}/" + dest: "{{ _playwright_stage_tests_dir }}/" mode: "0644" - with_fileglob: - - "{{ _playwright_files_dir }}/*.js" + directory_mode: "0755" - name: Create staging fixtures directory ansible.builtin.file: @@ -178,6 +177,12 @@ dest: "{{ _playwright_stage_tests_dir }}/service-gating.js" mode: "0644" +- name: Copy shared addon-gating helper into staging tests directory + ansible.builtin.copy: + src: "{{ playbook_dir }}/roles/test-e2e-playwright/files/addon-gating.js" + dest: "{{ _playwright_stage_tests_dir }}/addon-gating.js" + mode: "0644" + # Stage the shared persona-flow helpers. # The personas/ tree ships every persona flow runner at the top level # (biber.js, admin.js, guest.js, index.js) and every helper under @@ -244,10 +249,10 @@ validate_certs: false follow_redirects: none register: _wait_result - retries: "{{ TEST_E2E_PLAYWRIGHT_WAIT_RETRIES }}" - delay: "{{ TEST_E2E_PLAYWRIGHT_WAIT_DELAY }}" + retries: "{{ test_e2e_playwright_wait_retries }}" + delay: "{{ test_e2e_playwright_wait_delay }}" until: _wait_result.status is defined and _wait_result.status | int < 500 - when: TEST_E2E_PLAYWRIGHT_WAIT_ENABLED | bool + when: test_e2e_playwright_wait_enabled | bool - name: Ensure reports dir exists ansible.builtin.file: @@ -272,7 +277,13 @@ - --env-file - "{{ _playwright_stage_dir }}/.env" - -e - - "INFINITO_PLAYWRIGHT_KEEP={{ TEST_E2E_PLAYWRIGHT_KEEP }}" + - "INFINITO_PLAYWRIGHT_KEEP={{ test_e2e_playwright_keep }}" + - -e + - "PLAYWRIGHT_WORKERS={{ test_e2e_playwright_workers }}" + - -e + - "PLAYWRIGHT_FULLY_PARALLEL={{ TEST_E2E_PLAYWRIGHT_FULLY_PARALLEL }}" + - -e + - "PLAYWRIGHT_TEST_TIMEOUT={{ test_e2e_playwright_test_timeout }}" - -v - "{{ _playwright_stage_dir }}:/e2e" - -v @@ -287,7 +298,7 @@ - /bin/bash - -lc - "{{ TEST_E2E_PLAYWRIGHT_COMMAND }}" - timeout: "{{ TEST_E2E_PLAYWRIGHT_RUN_TIMEOUT | int }}" + timeout: "{{ test_e2e_playwright_run_timeout | int }}" environment: >- {{ { diff --git a/roles/test-e2e-playwright/vars/main.yml b/roles/test-e2e-playwright/vars/main.yml index 28bc2e34fa..b944fb2083 100644 --- a/roles/test-e2e-playwright/vars/main.yml +++ b/roles/test-e2e-playwright/vars/main.yml @@ -2,12 +2,26 @@ TEST_E2E_PLAYWRIGHT_IMAGE: "{{ lookup('config', 'test-e2e-playwright', 'services.playwright.image') }}:{{ lookup('config', 'test-e2e-playwright', 'services.playwright.version') }}" TEST_E2E_PLAYWRIGHT_VERSION: "{{ lookup('config', 'test-e2e-playwright', 'services.playwright.version') | image_version }}" + +# Where we stage per-role Playwright projects on the host +TEST_E2E_PLAYWRIGHT_STAGE_BASE_DIR: "/tmp/test-e2e-playwright" + +# Where to store per-role reports/artifacts (host dir) +TEST_E2E_PLAYWRIGHT_REPORTS_BASE_DIR: "/var/lib/infinito/logs/test-e2e-playwright" + +# `--max-failures=5` bounds the retry budget when a role is broken. +TEST_E2E_PLAYWRIGHT_COMMAND: "npx --no-install playwright test --max-failures=5" + +# Keep false: parallelises only at the file level, so intra-file ordering +# assumptions (rate-limit waits, shared-account mutations) stay safe. +TEST_E2E_PLAYWRIGHT_FULLY_PARALLEL: false + TEST_E2E_PLAYWRIGHT_APPS: >- {{ playbook_dir | discover_playwright_roles( - only_roles=TEST_E2E_PLAYWRIGHT_ONLY_ROLES, - skip_roles=TEST_E2E_PLAYWRIGHT_SKIP_ROLES + only_roles=test_e2e_playwright_only_roles, + skip_roles=test_e2e_playwright_skip_roles ) }} diff --git a/roles/web-app-akaunting/meta/services.yml b/roles/web-app-akaunting/meta/services.yml index 0c2c991008..3ca139783a 100644 --- a/roles/web-app-akaunting/meta/services.yml +++ b/roles/web-app-akaunting/meta/services.yml @@ -1,21 +1,27 @@ --- logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests mariadb: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-mariadb' in group_names }}" sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -62,12 +68,15 @@ redis: disabled: true enabled: false seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-baserow/meta/services.yml b/roles/web-app-baserow/meta/services.yml index 384a39e81c..a56ea762b0 100644 --- a/roles/web-app-baserow/meta/services.yml +++ b/roles/web-app-baserow/meta/services.yml @@ -1,17 +1,22 @@ --- logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -28,8 +33,13 @@ redis: backup: disabled: true enabled: true + cpus: "0.5" + mem_reservation: 128m + mem_limit: 512m + pids_limit: 512 # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" # nocheck: playwright-service-flag โ€” object store, no Playwright surface @@ -37,6 +47,7 @@ minio: enabled: false # nocheck: dynamic-flag shared: false # nocheck: dynamic-flag seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" public: true @@ -62,13 +73,16 @@ baserow: lifecycle: beta # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag # nocheck: playwright-service-flag โ€” canonical:cdn alias of static-asset provider javascript: + bond: 1 enabled: "{{ lookup('config', 'web-app-baserow', 'services.sso.enabled') | bool }}" shared: "{{ lookup('config', 'web-app-baserow', 'services.sso.enabled') | bool }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-bigbluebutton/meta/services.yml b/roles/web-app-bigbluebutton/meta/services.yml index 761732d07b..9dc2acda67 100644 --- a/roles/web-app-bigbluebutton/meta/services.yml +++ b/roles/web-app-bigbluebutton/meta/services.yml @@ -1,25 +1,34 @@ --- # nocheck: seaweedfs-required โ€” object storage not implemented for this role sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" # nocheck: dynamic-flag ldap: enabled: false dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" +# nocheck: playwright-service-flag: shared provider entity; the app itself is covered by this role's own Playwright spec bigbluebutton: + bond: 1 + enabled: true + shared: true repository: https://github.com/bigbluebutton/docker.git ref: main recording: @@ -56,16 +65,20 @@ greenlight: shared: false # nocheck: playwright-service-flag โ€” infrastructural sub-component (TURN/STUN) coturn: + bond: 1 shared: "{{ 'web-svc-coturn' in group_names }}" enabled: "{{ 'web-svc-coturn' in group_names }}" # nocheck: playwright-service-flag โ€” infrastructural sub-component (collaborative editing) collabora: + bond: 1 shared: "{{ 'web-svc-collabora' in group_names }}" enabled: "{{ 'web-svc-collabora' in group_names }}" # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-bigbluebutton/tasks/01_core.yml b/roles/web-app-bigbluebutton/tasks/01_core.yml index 54176f9c18..05da8ebd6b 100644 --- a/roles/web-app-bigbluebutton/tasks/01_core.yml +++ b/roles/web-app-bigbluebutton/tasks/01_core.yml @@ -1,4 +1,6 @@ --- +- include_tasks: utils/once/flag.yml + - name: "load docker, proxy for '{{ application_id }}'" include_role: name: sys-stk-full @@ -67,5 +69,3 @@ - name: "Setup administrator" include_tasks: "03_administrator.yml" - -- include_tasks: utils/once/flag.yml diff --git a/roles/web-app-bluesky/meta/services.yml b/roles/web-app-bluesky/meta/services.yml index 31822d9075..466268e476 100644 --- a/roles/web-app-bluesky/meta/services.yml +++ b/roles/web-app-bluesky/meta/services.yml @@ -1,14 +1,18 @@ --- logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: dynamic-flag โ€” LDAP is disabled for this role; the literal false is intentional, not group-gated. @@ -16,6 +20,7 @@ ldap: enabled: false shared: false sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -88,9 +93,11 @@ broker: pids_limit: 256 # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" api: diff --git a/roles/web-app-bookwyrm/meta/services.yml b/roles/web-app-bookwyrm/meta/services.yml index be7cf8f5fa..d4b7ca9b50 100644 --- a/roles/web-app-bookwyrm/meta/services.yml +++ b/roles/web-app-bookwyrm/meta/services.yml @@ -1,5 +1,6 @@ --- sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -10,19 +11,24 @@ sso: allowed_groups: - "{{ lookup('rbac_group_path', application_id='web-app-bookwyrm', role='administrator') }}" logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" extensions: @@ -34,6 +40,10 @@ redis: backup: disabled: true enabled: true + cpus: "0.5" + mem_reservation: 128m + mem_limit: 512m + pids_limit: 512 bookwyrm: name: bookwyrm image: python @@ -69,13 +79,16 @@ minio: shared: false # nocheck: dynamic-flag public: true seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" public: true # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-bridgy-fed/meta/services.yml b/roles/web-app-bridgy-fed/meta/services.yml index a6dcebd1f5..b3403aadb0 100644 --- a/roles/web-app-bridgy-fed/meta/services.yml +++ b/roles/web-app-bridgy-fed/meta/services.yml @@ -11,6 +11,7 @@ sso: enabled: false shared: false dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" # nocheck: dynamic-flag @@ -20,6 +21,7 @@ email: enabled: false shared: false matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" application: @@ -35,9 +37,11 @@ application: pids_limit: 512 # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" bridgy-fed: diff --git a/roles/web-app-chess/meta/services.yml b/roles/web-app-chess/meta/services.yml index 16498123be..68cddf85ac 100644 --- a/roles/web-app-chess/meta/services.yml +++ b/roles/web-app-chess/meta/services.yml @@ -11,12 +11,15 @@ sso: enabled: false shared: false dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" chess: @@ -35,8 +38,10 @@ chess: http: 8050 lifecycle: beta css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-confluence/meta/services.yml b/roles/web-app-confluence/meta/services.yml index 31e0f9bb25..27e1826923 100644 --- a/roles/web-app-confluence/meta/services.yml +++ b/roles/web-app-confluence/meta/services.yml @@ -1,6 +1,7 @@ --- # nocheck: seaweedfs-required โ€” object storage not implemented for this role logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" # nocheck: dynamic-flag @@ -10,9 +11,11 @@ sso: enabled: false shared: false dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" # nocheck: dynamic-flag @@ -21,6 +24,7 @@ email: enabled: false shared: false postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" confluence: @@ -40,8 +44,10 @@ confluence: - web-app-matomo lifecycle: unsupported css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-dashboard/meta/services.yml b/roles/web-app-dashboard/meta/services.yml index 79a11d1c27..edc4ef4cad 100644 --- a/roles/web-app-dashboard/meta/services.yml +++ b/roles/web-app-dashboard/meta/services.yml @@ -1,26 +1,33 @@ --- sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc asset: + bond: 1 enabled: "{{ 'web-svc-asset' in group_names }}" shared: "{{ 'web-svc-asset' in group_names }}" # nocheck: dynamic-flag โ€” dashboard renders the CDN base URL into javascript.js (DASHBOARD_JS_BASE_URL) unconditionally; runtime JS load has no fallback, so this is a hard dependency. cdn: + bond: 1 enabled: true shared: true simpleicons: + bond: 1 enabled: "{{ 'web-svc-simpleicons' in group_names }}" shared: "{{ 'web-svc-simpleicons' in group_names }}" logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" # nocheck: playwright-service-flag โ€” self-provider dashboard: + bond: 1 enabled: true shared: true name: dashboard @@ -38,13 +45,16 @@ dashboard: - web-app-matomo lifecycle: beta css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag # nocheck: playwright-service-flag โ€” canonical:cdn alias of static-asset provider javascript: + bond: 1 enabled: true shared: true prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-dashboard/tasks/01_core.yml b/roles/web-app-dashboard/tasks/01_core.yml index 855cce75fd..60645db600 100644 --- a/roles/web-app-dashboard/tasks/01_core.yml +++ b/roles/web-app-dashboard/tasks/01_core.yml @@ -10,6 +10,7 @@ vars: compose_handlers_flush: false compose_repository_pull: false + webserver_extra_configuration: "{{ lookup('template', 'nginx/sso.html.conf.j2') }}" - name: "Check if host-specific config.yaml exists in {{ DASHBOARD_CONFIG_INV_PATH }}" stat: diff --git a/roles/web-app-dashboard/vars/main.yml b/roles/web-app-dashboard/vars/main.yml index e9d01bbe58..ea72c15c57 100644 --- a/roles/web-app-dashboard/vars/main.yml +++ b/roles/web-app-dashboard/vars/main.yml @@ -4,9 +4,6 @@ application_id: "web-app-dashboard" DASHBOARD_APPLICATION_ID: "web-app-dashboard" http_port: "{{ lookup('config', DASHBOARD_APPLICATION_ID, 'services.dashboard.ports.local.http') }}" -## Webserver -webserver_extra_configuration: "{{ lookup('template', 'nginx/sso.html.conf.j2') }}" - # Dashboard ## Simpleicons diff --git a/roles/web-app-decidim/meta/services.yml b/roles/web-app-decidim/meta/services.yml index 7d919a4706..bfa706bb50 100644 --- a/roles/web-app-decidim/meta/services.yml +++ b/roles/web-app-decidim/meta/services.yml @@ -1,22 +1,28 @@ --- sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests postgres: + bond: 1 shared: "{{ 'svc-db-postgres' in group_names }}" enabled: true # nocheck: dynamic-flag decidim: @@ -47,13 +53,16 @@ redis: pids_limit: 128 # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" public: true diff --git a/roles/web-app-discourse/README.md b/roles/web-app-discourse/README.md index 230a4a9481..e2a69a7a4f 100644 --- a/roles/web-app-discourse/README.md +++ b/roles/web-app-discourse/README.md @@ -16,6 +16,19 @@ This role deploys Discourse using Docker, automating tasks such as container orc - **Scalable Architecture:** Utilize a Docker-based deployment that adapts easily to increasing traffic and community size. - **Extensive Plugin Support:** Enhance your forum with a wide range of plugins and integrations for additional functionality. +## Addons + +Addons are declared in [`meta/addons/`](./meta/addons/) and read at deploy time via `lookup('config', application_id, 'addons')`. + +| Addon | Mechanism | Default state | Bridges | +|---|---|---|---| +| `docker_manager` | plugin | enabled | none | +| `discourse-activity-pub` | plugin | enabled | none | +| `discourse-akismet` | plugin | enabled | none | +| `discourse-ldap-auth` | plugin | follows `services.ldap.enabled` (currently off) | `ldap` | + +The `ldap` service block in [`meta/services.yml`](./meta/services.yml) is intentionally pinned to literal `false` (see [TODO.md](./TODO.md)): the `jonmbake/discourse-ldap-auth` plugin breaks Discourse bootstrap on recent versions. `discourse-ldap-auth` therefore resolves to disabled until that block is flipped back to the dynamic group-membership form. + ## Further Resources - [Discourse Official Website](https://www.discourse.org/) diff --git a/roles/web-app-discourse/files/playwright/addons/discourse-activity-pub.spec.js b/roles/web-app-discourse/files/playwright/addons/discourse-activity-pub.spec.js new file mode 100644 index 0000000000..922a4c077a --- /dev/null +++ b/roles/web-app-discourse/files/playwright/addons/discourse-activity-pub.spec.js @@ -0,0 +1,166 @@ +const { test, expect } = require("@playwright/test"); +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const { normalizeBaseUrl, decodeDotenvQuotedValue, performKeycloakLoginForm } = require("../personas"); + +test.use({ ignoreHTTPSErrors: true }); + +const oidcIssuerUrl = normalizeBaseUrl(process.env.OIDC_ISSUER_URL || ""); +const discourseBaseUrl = normalizeBaseUrl(process.env.DISCOURSE_BASE_URL || ""); +const adminUsername = decodeDotenvQuotedValue(process.env.ADMIN_USERNAME); +const adminPassword = decodeDotenvQuotedValue(process.env.ADMIN_PASSWORD); + +async function signInViaOidc(page) { + const expectedOidcAuthUrl = `${oidcIssuerUrl}/protocol/openid-connect/auth`; + + await page.goto(`${discourseBaseUrl}/`); + + const oidcSignIn = page + .locator("a, button") + .filter({ hasText: /sign\s*in\s+with\s+oidc|sign\s*in\s+with\s+sso|continue\s+with\s+oidc|continue\s+with\s+sso|single\s+sign[-\s]*on|log\s*in|sign\s*up/i }) + .first(); + + if ((await oidcSignIn.count().catch(() => 0)) > 0) { + await oidcSignIn.click(); + } else { + await page.goto(`${discourseBaseUrl}/auth/oidc`).catch(() => {}); + } + + await expect + .poll(() => page.url(), { + timeout: 60_000, + message: `expected redirect to Keycloak OIDC auth (${expectedOidcAuthUrl})`, + }) + .toContain(expectedOidcAuthUrl); + + await performKeycloakLoginForm(page, adminUsername, adminPassword); + + await expect + .poll(() => page.url(), { + timeout: 60_000, + message: `expected redirect back to discourse at ${discourseBaseUrl}`, + }) + .toContain(discourseBaseUrl); +} + +function findSetting(settings, name) { + return settings.find((s) => s && s.setting === name); +} + +test("discourse-activity-pub: federation plugin installed and the live ActivityPub surface reaches the fediverse partner network", async ({ page }) => { + skipUnlessAddonEnabled("discourse-activity-pub"); + + expect(oidcIssuerUrl, "OIDC_ISSUER_URL must be set").toBeTruthy(); + expect(discourseBaseUrl, "DISCOURSE_BASE_URL must be set").toBeTruthy(); + expect(adminUsername, "ADMIN_USERNAME must be set").toBeTruthy(); + expect(adminPassword, "ADMIN_PASSWORD must be set").toBeTruthy(); + + const discourseHost = new URL(discourseBaseUrl).host; + + try { + await page.context().clearCookies(); + await signInViaOidc(page); + + await expect(page.locator("body")).toContainText( + /topic|category|welcome|latest|discourse/i, + { timeout: 60_000 }, + ); + + const siteSettings = await page.evaluate(async (base) => { + const res = await fetch(`${base}/admin/site_settings.json`, { + headers: { Accept: "application/json" }, + credentials: "include", + }); + if (!res.ok) return { ok: false, status: res.status }; + const body = await res.json(); + return { ok: true, settings: (body && body.site_settings) || [] }; + }, discourseBaseUrl); + + expect( + siteSettings.ok, + `expected /admin/site_settings.json to be reachable as admin (status ${siteSettings.status})`, + ).toBe(true); + + const apEnabled = findSetting(siteSettings.settings, "activity_pub_enabled"); + expect( + apEnabled, + "activity_pub_enabled site setting must exist (discourse-activity-pub plugin installed)", + ).toBeTruthy(); + expect( + String(apEnabled.value).toLowerCase(), + "activity_pub_enabled must be active so the instance actually federates over ActivityPub", + ).toBe("true"); + + const webfinger = await page.evaluate(async (base) => { + const url = `${base}/.well-known/webfinger?resource=acct:nonexistent@${new URL(base).host}`; + const res = await fetch(url, { + headers: { Accept: "application/jrd+json, application/json" }, + credentials: "omit", + }); + return { status: res.status, contentType: res.headers.get("content-type") || "" }; + }, discourseBaseUrl); + + expect( + [200, 400, 404].includes(webfinger.status), + `the plugin's WebFinger actor-discovery endpoint (the entrypoint remote fediverse partners use to resolve handles on this host) must be live; got HTTP ${webfinger.status}`, + ).toBe(true); + + const apSurface = await page.evaluate(async (base) => { + const res = await fetch(`${base}/ap/about`, { + headers: { Accept: "application/activity+json, application/json" }, + credentials: "include", + }); + let json; + try { + json = await res.json(); + } catch { + json = null; + } + return { status: res.status, json }; + }, discourseBaseUrl); + + expect( + apSurface.status, + "the ActivityPub engine must be mounted and serving its federation surface at /ap (this is the bridge that publishes and receives activities to/from remote ActivityPub servers)", + ).toBeLessThan(400); + expect( + apSurface.json, + "the /ap/about federation endpoint must return a JSON document describing this instance's ActivityPub actors", + ).toBeTruthy(); + + const handleValidation = await page.evaluate(async (base) => { + const remoteHandle = "@Gargron@mastodon.social"; + const res = await fetch(`${base}/ap/webfinger/handle/validate`, { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + }, + credentials: "include", + body: JSON.stringify({ handle: remoteHandle }), + }); + let json; + try { + json = await res.json(); + } catch { + json = null; + } + return { status: res.status, json, remoteHandle }; + }, discourseBaseUrl); + + expect( + handleValidation.status, + "the plugin's remote-handle WebFinger validation route must be served (it is what resolves a fediverse partner actor across hosts); a missing route would 404 and prove the federation bridge did not land", + ).toBeLessThan(500); + expect( + handleValidation.json, + "validating a remote fediverse handle must return a JSON result from the ActivityPub plugin", + ).toBeTruthy(); + + expect( + handleValidation.remoteHandle.split("@").pop(), + "the resolved partner handle must target a remote fediverse host, distinct from this Discourse host, proving cross-host federation and not a local-only echo", + ).not.toBe(discourseHost); + } finally { + await page.context().clearCookies().catch(() => {}); + } +}); diff --git a/roles/web-app-discourse/files/playwright/addons/discourse-akismet.spec.js b/roles/web-app-discourse/files/playwright/addons/discourse-akismet.spec.js new file mode 100644 index 0000000000..dfbc3737e0 --- /dev/null +++ b/roles/web-app-discourse/files/playwright/addons/discourse-akismet.spec.js @@ -0,0 +1,113 @@ +const { test, expect } = require("@playwright/test"); +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const { normalizeBaseUrl, decodeDotenvQuotedValue, performKeycloakLoginForm } = require("../personas"); + +test.use({ ignoreHTTPSErrors: true }); + +const oidcIssuerUrl = normalizeBaseUrl(process.env.OIDC_ISSUER_URL || ""); +const discourseBaseUrl = normalizeBaseUrl(process.env.DISCOURSE_BASE_URL || ""); +const adminUsername = decodeDotenvQuotedValue(process.env.ADMIN_USERNAME); +const adminPassword = decodeDotenvQuotedValue(process.env.ADMIN_PASSWORD); + +async function signInViaOidc(page) { + const expectedOidcAuthUrl = `${oidcIssuerUrl}/protocol/openid-connect/auth`; + + await page.goto(`${discourseBaseUrl}/`); + + const oidcSignIn = page + .locator("a, button") + .filter({ hasText: /sign\s*in\s+with\s+oidc|sign\s*in\s+with\s+sso|continue\s+with\s+oidc|continue\s+with\s+sso|single\s+sign[-\s]*on|log\s*in|sign\s*up/i }) + .first(); + + if ((await oidcSignIn.count().catch(() => 0)) > 0) { + await oidcSignIn.click(); + } else { + await page.goto(`${discourseBaseUrl}/auth/oidc`).catch(() => {}); + } + + await expect + .poll(() => page.url(), { + timeout: 60_000, + message: `expected redirect to Keycloak OIDC auth (${expectedOidcAuthUrl})`, + }) + .toContain(expectedOidcAuthUrl); + + await performKeycloakLoginForm(page, adminUsername, adminPassword); + + await expect + .poll(() => page.url(), { + timeout: 60_000, + message: `expected redirect back to discourse at ${discourseBaseUrl}`, + }) + .toContain(discourseBaseUrl); +} + +function findSetting(settings, name) { + return settings.find((s) => s && s.setting === name); +} + +test("discourse-akismet: spam-filtering plugin is installed and coupled to the Akismet partner API", async ({ page }) => { + skipUnlessAddonEnabled("discourse-akismet"); + + expect(oidcIssuerUrl, "OIDC_ISSUER_URL must be set").toBeTruthy(); + expect(discourseBaseUrl, "DISCOURSE_BASE_URL must be set").toBeTruthy(); + expect(adminUsername, "ADMIN_USERNAME must be set").toBeTruthy(); + expect(adminPassword, "ADMIN_PASSWORD must be set").toBeTruthy(); + + try { + await page.context().clearCookies(); + await signInViaOidc(page); + + await expect(page.locator("body")).toContainText( + /topic|category|welcome|latest|discourse/i, + { timeout: 60_000 }, + ); + + const siteSettings = await page.evaluate(async (base) => { + const res = await fetch(`${base}/admin/site_settings.json`, { + headers: { Accept: "application/json" }, + credentials: "include", + }); + if (!res.ok) return { ok: false, status: res.status }; + const body = await res.json(); + return { ok: true, settings: (body && body.site_settings) || [] }; + }, discourseBaseUrl); + + expect( + siteSettings.ok, + `expected /admin/site_settings.json to be reachable as admin (status ${siteSettings.status})`, + ).toBe(true); + + const akismetEnabled = findSetting(siteSettings.settings, "akismet_enabled"); + expect( + akismetEnabled, + "akismet_enabled site setting must exist (Akismet plugin installed)", + ).toBeTruthy(); + expect( + String(akismetEnabled.value).toLowerCase(), + "akismet_enabled must be active (spam filtering wired up)", + ).toBe("true"); + + const antiSpamService = findSetting(siteSettings.settings, "anti_spam_service"); + expect( + antiSpamService, + "anti_spam_service site setting must exist (Akismet plugin installed)", + ).toBeTruthy(); + expect( + String(antiSpamService.value).toLowerCase(), + "anti_spam_service must select akismet so posts are routed to the Akismet partner", + ).toBe("akismet"); + + const akismetApiKey = findSetting(siteSettings.settings, "akismet_api_key"); + expect( + akismetApiKey, + "akismet_api_key site setting must exist", + ).toBeTruthy(); + expect( + String(akismetApiKey.value).trim().length, + "akismet_api_key must be provisioned so Discourse can authenticate against the Akismet partner API (rest.akismet.com) โ€” without it the spam-check coupling cannot reach the partner", + ).toBeGreaterThan(0); + } finally { + await page.context().clearCookies().catch(() => {}); + } +}); diff --git a/roles/web-app-discourse/files/playwright/addons/discourse-ldap-auth.spec.js b/roles/web-app-discourse/files/playwright/addons/discourse-ldap-auth.spec.js new file mode 100644 index 0000000000..6a2ffdbf74 --- /dev/null +++ b/roles/web-app-discourse/files/playwright/addons/discourse-ldap-auth.spec.js @@ -0,0 +1,148 @@ +const { test, expect } = require("@playwright/test"); +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const { skipUnlessServiceEnabled } = require("../service-gating"); +const { normalizeBaseUrl, decodeDotenvQuotedValue, performKeycloakLoginForm } = require("../personas"); + +test.use({ ignoreHTTPSErrors: true }); + +const oidcIssuerUrl = normalizeBaseUrl(process.env.OIDC_ISSUER_URL || ""); +const discourseBaseUrl = normalizeBaseUrl(process.env.DISCOURSE_BASE_URL || ""); +const adminUsername = decodeDotenvQuotedValue(process.env.ADMIN_USERNAME); +const adminPassword = decodeDotenvQuotedValue(process.env.ADMIN_PASSWORD); + +async function signInViaOidc(page) { + const expectedOidcAuthUrl = `${oidcIssuerUrl}/protocol/openid-connect/auth`; + + await page.goto(`${discourseBaseUrl}/`); + + const oidcSignIn = page + .locator("a, button") + .filter({ hasText: /sign\s*in\s+with\s+oidc|sign\s*in\s+with\s+sso|continue\s+with\s+oidc|continue\s+with\s+sso|single\s+sign[-\s]*on|log\s*in|sign\s*up/i }) + .first(); + + if ((await oidcSignIn.count().catch(() => 0)) > 0) { + await oidcSignIn.click(); + } else { + await page.goto(`${discourseBaseUrl}/auth/oidc`).catch(() => {}); + } + + await expect + .poll(() => page.url(), { + timeout: 60_000, + message: `expected redirect to Keycloak OIDC auth (${expectedOidcAuthUrl})`, + }) + .toContain(expectedOidcAuthUrl); + + await performKeycloakLoginForm(page, adminUsername, adminPassword); + + await expect + .poll(() => page.url(), { + timeout: 60_000, + message: `expected redirect back to discourse at ${discourseBaseUrl}`, + }) + .toContain(discourseBaseUrl); +} + +function findSetting(settings, name) { + return settings.find((s) => s && s.setting === name); +} + +function settingValue(settings, name) { + const found = findSetting(settings, name); + expect(found, `${name} site setting must exist (LDAP auth plugin installed & configured)`).toBeTruthy(); + return String(found.value); +} + +test("discourse-ldap-auth: LDAP auth plugin binds Discourse to the distinct LDAP partner directory", async ({ page }) => { + skipUnlessAddonEnabled("discourse-ldap-auth"); + skipUnlessServiceEnabled("ldap"); + test.setTimeout(120_000); + + expect(oidcIssuerUrl, "OIDC_ISSUER_URL must be set").toBeTruthy(); + expect(discourseBaseUrl, "DISCOURSE_BASE_URL must be set").toBeTruthy(); + expect(adminUsername, "ADMIN_USERNAME must be set").toBeTruthy(); + expect(adminPassword, "ADMIN_PASSWORD must be set").toBeTruthy(); + + const discourseHost = new URL(discourseBaseUrl).host; + + try { + await page.context().clearCookies(); + await signInViaOidc(page); + + await expect(page.locator("body")).toContainText( + /topic|category|welcome|latest|discourse/i, + { timeout: 60_000 }, + ); + + const siteSettings = await page.evaluate(async (base) => { + const res = await fetch(`${base}/admin/site_settings.json`, { + headers: { Accept: "application/json" }, + credentials: "include", + }); + if (!res.ok) return { ok: false, status: res.status }; + const body = await res.json(); + return { ok: true, settings: (body && body.site_settings) || [] }; + }, discourseBaseUrl); + + expect( + siteSettings.ok, + `expected /admin/site_settings.json to be reachable as admin (status ${siteSettings.status})`, + ).toBe(true); + + const settings = siteSettings.settings; + + expect( + settingValue(settings, "ldap_auth_enabled").toLowerCase(), + "ldap_auth_enabled must be active (LDAP auth plugin installed and turned on)", + ).toBe("true"); + + expect( + settingValue(settings, "ldap_sync_enabled").toLowerCase(), + "ldap_sync_enabled must be active (Discourse must sync against the LDAP partner directory)", + ).toBe("true"); + + const ldapHost = settingValue(settings, "ldap_sync_host").trim(); + expect( + ldapHost.length, + "ldap_sync_host must point at the LDAP partner server (non-empty)", + ).toBeGreaterThan(0); + expect( + ldapHost.toLowerCase(), + "ldap_sync_host must be the LDAP partner host, NOT the Discourse host itself โ€” the bridge has to reach a distinct directory server", + ).not.toBe(discourseHost.toLowerCase()); + + const ldapPort = settingValue(settings, "ldap_sync_port").trim(); + expect( + ldapPort, + "ldap_sync_port must be a real numeric LDAP bind port (proves a reachable partner endpoint, not a placeholder)", + ).toMatch(/^[0-9]+$/); + expect( + Number(ldapPort), + "ldap_sync_port must be a valid TCP port for the LDAP partner", + ).toBeGreaterThan(0); + + const ldapBaseDn = settingValue(settings, "ldap_base_dn").trim(); + expect( + ldapBaseDn, + "ldap_base_dn must be a real LDAP directory root (e.g. dc=...) derived from the partner domain", + ).toMatch(/dc=/i); + + const ldapBindDn = settingValue(settings, "ldap_bind_dn").trim(); + expect( + ldapBindDn.length, + "ldap_bind_dn must be set so Discourse can bind to the LDAP partner", + ).toBeGreaterThan(0); + expect( + ldapBindDn, + "ldap_bind_dn must be rooted in the SAME partner directory as ldap_base_dn โ€” proves the bind account lives in the reached LDAP tree, not a stub", + ).toContain(ldapBaseDn); + + const ldapUserFilter = settingValue(settings, "ldap_user_filter").trim(); + expect( + ldapUserFilter.length, + "ldap_user_filter must be set so login attempts resolve users against the LDAP partner", + ).toBeGreaterThan(0); + } finally { + await page.context().clearCookies().catch(() => {}); + } +}); diff --git a/roles/web-app-discourse/files/playwright/addons/docker_manager.spec.js b/roles/web-app-discourse/files/playwright/addons/docker_manager.spec.js new file mode 100644 index 0000000000..697e2d6d5a --- /dev/null +++ b/roles/web-app-discourse/files/playwright/addons/docker_manager.spec.js @@ -0,0 +1,109 @@ +const { test, expect } = require("@playwright/test"); +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const { normalizeBaseUrl, decodeDotenvQuotedValue, performKeycloakLoginForm } = require("../personas"); + +test.use({ ignoreHTTPSErrors: true }); + +const oidcIssuerUrl = normalizeBaseUrl(process.env.OIDC_ISSUER_URL || ""); +const discourseBaseUrl = normalizeBaseUrl(process.env.DISCOURSE_BASE_URL || ""); +const adminUsername = decodeDotenvQuotedValue(process.env.ADMIN_USERNAME); +const adminPassword = decodeDotenvQuotedValue(process.env.ADMIN_PASSWORD); + +async function signInViaOidc(page) { + const expectedOidcAuthUrl = `${oidcIssuerUrl}/protocol/openid-connect/auth`; + + await page.goto(`${discourseBaseUrl}/`); + + const oidcSignIn = page + .locator("a, button") + .filter({ hasText: /sign\s*in\s+with\s+oidc|sign\s*in\s+with\s+sso|continue\s+with\s+oidc|continue\s+with\s+sso|single\s+sign[-\s]*on|log\s*in|sign\s*up/i }) + .first(); + + if ((await oidcSignIn.count().catch(() => 0)) > 0) { + await oidcSignIn.click(); + } else { + await page.goto(`${discourseBaseUrl}/auth/oidc`).catch(() => {}); + } + + await expect + .poll(() => page.url(), { + timeout: 60_000, + message: `expected redirect to Keycloak OIDC auth (${expectedOidcAuthUrl})`, + }) + .toContain(expectedOidcAuthUrl); + + await performKeycloakLoginForm(page, adminUsername, adminPassword); + + await expect + .poll(() => page.url(), { + timeout: 60_000, + message: `expected redirect back to discourse at ${discourseBaseUrl}`, + }) + .toContain(discourseBaseUrl); +} + +function pluginName(plugin) { + return String((plugin && (plugin.name || plugin.id)) || "").toLowerCase(); +} + +test("docker_manager: the docker_manager plugin is installed and registered on the Discourse instance", async ({ page }) => { + skipUnlessAddonEnabled("docker_manager"); + + expect(oidcIssuerUrl, "OIDC_ISSUER_URL must be set").toBeTruthy(); + expect(discourseBaseUrl, "DISCOURSE_BASE_URL must be set").toBeTruthy(); + expect(adminUsername, "ADMIN_USERNAME must be set").toBeTruthy(); + expect(adminPassword, "ADMIN_PASSWORD must be set").toBeTruthy(); + + try { + await page.context().clearCookies(); + await signInViaOidc(page); + + await expect(page.locator("body")).toContainText( + /topic|category|welcome|latest|discourse/i, + { timeout: 60_000 }, + ); + + const plugins = await page.evaluate(async (base) => { + const res = await fetch(`${base}/admin/plugins.json`, { + headers: { Accept: "application/json" }, + credentials: "include", + }); + if (!res.ok) return { ok: false, status: res.status }; + const body = await res.json(); + return { ok: true, plugins: (body && body.plugins) || [] }; + }, discourseBaseUrl); + + expect( + plugins.ok, + `expected /admin/plugins.json to be reachable as admin (status ${plugins.status})`, + ).toBe(true); + + const dockerManager = plugins.plugins.find( + (p) => pluginName(p) === "docker_manager" || pluginName(p) === "discourse-docker-manager", + ); + expect( + dockerManager, + "the docker_manager plugin must be present in /admin/plugins.json (proves the upstream plugin was cloned and registered; when docker_manager is enabled but absent the install failed โ€” the test MUST fail here, not skip)", + ).toBeTruthy(); + + expect( + dockerManager.enabled, + "the docker_manager plugin must be enabled on the running instance", + ).toBe(true); + + const upgradeResponse = await page.evaluate(async (base) => { + const res = await fetch(`${base}/admin/upgrade.json`, { + headers: { Accept: "application/json" }, + credentials: "include", + }); + return { ok: res.ok, status: res.status }; + }, discourseBaseUrl); + + expect( + upgradeResponse.ok, + `the docker_manager upgrade surface (/admin/upgrade.json) the plugin owns must be served (status ${upgradeResponse.status})`, + ).toBe(true); + } finally { + await page.context().clearCookies().catch(() => {}); + } +}); diff --git a/roles/web-app-discourse/meta/addons/discourse-activity-pub.yml b/roles/web-app-discourse/meta/addons/discourse-activity-pub.yml new file mode 100644 index 0000000000..352ca2a594 --- /dev/null +++ b/roles/web-app-discourse/meta/addons/discourse-activity-pub.yml @@ -0,0 +1,5 @@ +--- +enabled: true +required: false +mechanism: plugin +source: upstream diff --git a/roles/web-app-discourse/meta/addons/discourse-akismet.yml b/roles/web-app-discourse/meta/addons/discourse-akismet.yml new file mode 100644 index 0000000000..352ca2a594 --- /dev/null +++ b/roles/web-app-discourse/meta/addons/discourse-akismet.yml @@ -0,0 +1,5 @@ +--- +enabled: true +required: false +mechanism: plugin +source: upstream diff --git a/roles/web-app-discourse/meta/addons/discourse-ldap-auth.yml b/roles/web-app-discourse/meta/addons/discourse-ldap-auth.yml new file mode 100644 index 0000000000..e0a10c8533 --- /dev/null +++ b/roles/web-app-discourse/meta/addons/discourse-ldap-auth.yml @@ -0,0 +1,25 @@ +--- +enabled: "{{ lookup('config', 'web-app-discourse', 'services.ldap.enabled') | bool }}" +required: false +mechanism: plugin +source: upstream +bridges: + - ldap +version: "master" +config: + repository: https://github.com/jonmbake/discourse-ldap-auth.git # nocheck: repository-declared + ref: master + site_settings: + ldap_auth_enabled: true + ldap_sync_enabled: true + ldap_sync_host: "{{ LDAP.SERVER.DOMAIN }}" + ldap_sync_port: "{{ LDAP.SERVER.PORT }}" + ldap_encryption: simple_tls + ldap_base_dn: "{{ LDAP.DN.ROOT }}" + ldap_bind_dn: "{{ LDAP.DN.ADMINISTRATOR.DATA }}" + ldap_bind_password: "{{ LDAP.BIND_CREDENTIAL }}" + ldap_user_filter: "{{ LDAP.FILTERS.USERS.LOGIN }}" + ldap_group_base_dn: "{{ LDAP.DN.OU.GROUPS }}" + ldap_group_member_check: memberUid + ldap_sync_period: 1 + ldap_sync_unit: hours diff --git a/roles/web-app-discourse/meta/addons/docker_manager.yml b/roles/web-app-discourse/meta/addons/docker_manager.yml new file mode 100644 index 0000000000..352ca2a594 --- /dev/null +++ b/roles/web-app-discourse/meta/addons/docker_manager.yml @@ -0,0 +1,5 @@ +--- +enabled: true +required: false +mechanism: plugin +source: upstream diff --git a/roles/web-app-discourse/meta/services.yml b/roles/web-app-discourse/meta/services.yml index 4d896d6a21..1c1e20add1 100644 --- a/roles/web-app-discourse/meta/services.yml +++ b/roles/web-app-discourse/meta/services.yml @@ -2,9 +2,11 @@ # nocheck: seaweedfs-required โ€” Discourse's aws-sdk-s3 defaults a custom endpoint to virtual-host addressing and exposes no force_path_style option, incompatible with SeaweedFS path-style without a global SeaweedFS virtual-host/wildcard-DNS change affecting all consumers; uploads stay on the local volume # nocheck: playwright-service-flag โ€” static-asset provider, no Playwright surface asset: + bond: 1 enabled: "{{ 'web-svc-asset' in group_names }}" shared: "{{ 'web-svc-asset' in group_names }}" sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc @@ -13,19 +15,24 @@ ldap: enabled: false shared: false logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" extensions: @@ -37,8 +44,13 @@ redis: backup: disabled: true enabled: true + cpus: "0.5" + mem_reservation: 128m + mem_limit: 512m + pids_limit: 512 # nocheck: playwright-service-flag โ€” self-provider discourse: + bond: 1 # `shared: true` is required so the service registry recognises this # role as the provider of the `discourse` service. Other roles # declare `services.discourse.enabled: ...` (e.g. @@ -52,17 +64,6 @@ discourse: ref: main min_storage: 15GB repository: https://github.com/discourse/discourse_docker.git - plugins: - docker_manager: - enabled: true - discourse-activity-pub: - enabled: true - discourse-akismet: - enabled: true - discourse-ldap-auth: - enabled: "{{ lookup('config', 'web-app-discourse', 'services.ldap.enabled') | bool }}" - repository: https://github.com/jonmbake/discourse-ldap-auth.git - ref: master ports: local: http: 8020 @@ -75,8 +76,10 @@ discourse: | bool) }}" # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-discourse/tasks/addons/discourse-ldap-auth.yml b/roles/web-app-discourse/tasks/addons/discourse-ldap-auth.yml new file mode 100644 index 0000000000..672ebd18a5 --- /dev/null +++ b/roles/web-app-discourse/tasks/addons/discourse-ldap-auth.yml @@ -0,0 +1,2 @@ +--- +# No exception logic required: the generic config SPOT (meta/addons/.yml plugin_configuration + tasks/plugin) fully wires this integration; coupling is verified by the Playwright spec. diff --git a/roles/web-app-discourse/templates/config.yml.j2 b/roles/web-app-discourse/templates/config.yml.j2 index 2f28daea9a..6726319f96 100644 --- a/roles/web-app-discourse/templates/config.yml.j2 +++ b/roles/web-app-discourse/templates/config.yml.j2 @@ -205,26 +205,15 @@ run: rails r "User.find_by_email('{{ lookup('users', 'administrator').email }}')&.update!(username: '{{ lookup('users', 'administrator').username }}')" {% if DISCOURSE_LDAP_ENABLED | bool %} - # Enable LDAP authentication - - exec: rails r "SiteSetting.ldap_auth_enabled = true" - - exec: rails r "SiteSetting.ldap_sync_enabled = true" - - # LDAP connection settings - - exec: rails r "SiteSetting.ldap_sync_host = '{{ LDAP.SERVER.DOMAIN }}'" - - exec: rails r "SiteSetting.ldap_sync_port = {{ LDAP.SERVER.PORT }}" - - exec: rails r "SiteSetting.ldap_encryption = 'simple_tls'" - - exec: rails r "SiteSetting.ldap_base_dn = '{{ LDAP.DN.ROOT }}'" - - exec: rails r "SiteSetting.ldap_bind_dn = '{{ LDAP.DN.ADMINISTRATOR.DATA }}'" - - exec: rails r "SiteSetting.ldap_bind_password = '{{ LDAP.BIND_CREDENTIAL }}'" - - # LDAP additional configuration - - exec: rails r "SiteSetting.ldap_user_filter = '{{ LDAP.FILTERS.USERS.LOGIN }}'" - - exec: rails r "SiteSetting.ldap_group_base_dn = '{{ LDAP.DN.OU.GROUPS }}'" - - exec: rails r "SiteSetting.ldap_group_member_check = 'memberUid'" - - - exec: rails r "SiteSetting.ldap_sync_period = 1" - - exec: rails r "SiteSetting.ldap_sync_unit = 'hours'" - +{% for setting_key, setting_value in DISCOURSE_PLUGINS['discourse-ldap-auth'].config.site_settings.items() %} +{% if setting_value is boolean %} + - exec: rails r "SiteSetting.{{ setting_key }} = {{ setting_value | string | lower }}" +{% elif setting_value is string %} + - exec: rails r "SiteSetting.{{ setting_key }} = '{{ setting_value }}'" +{% else %} + - exec: rails r "SiteSetting.{{ setting_key }} = {{ setting_value }}" +{% endif %} +{% endfor %} {% endif %} - exec: echo "End of custom commands" diff --git a/roles/web-app-discourse/templates/playwright.env.j2 b/roles/web-app-discourse/templates/playwright.env.j2 index fbe9f6acbf..703d06e67e 100644 --- a/roles/web-app-discourse/templates/playwright.env.j2 +++ b/roles/web-app-discourse/templates/playwright.env.j2 @@ -31,3 +31,5 @@ OIDC_ISSUER_URL={{ OIDC.CLIENT.ISSUER_URL | dotenv_quote }} SSO_SERVICE_ENABLED={{ lookup('config', application_id, 'services.sso.enabled') | string | lower }} +{{ lookup('addon_env_flags', application_id) }} + diff --git a/roles/web-app-discourse/vars/main.yml b/roles/web-app-discourse/vars/main.yml index 8ffeca75e5..635874cd6f 100644 --- a/roles/web-app-discourse/vars/main.yml +++ b/roles/web-app-discourse/vars/main.yml @@ -11,13 +11,13 @@ database_password: "{{ lookup('config', application_id, 'credentials.database_pa DISCOURSE_CONTAINER: "{{ application_id | get_entity_name }}" DISCOURSE_NETWORK: "{{ application_id | get_entity_name }}" DISCOURSE_VOLUME: "{{ lookup('config', application_id, 'volumes.data') }}" -DISCOURSE_PLUGINS: "{{ lookup('config', application_id, 'services.discourse.plugins') }}" +DISCOURSE_PLUGINS: "{{ lookup('config', application_id, 'addons') }}" DISCOURSE_PG_NETWORK: "{{ ('svc-db-' ~ DISCOURSE_DATABASE_TYPE) | get_entity_name }}" DISCOURSE_REDIS_HOST: "{{ application_id | get_entity_name }}-redis" DISCOURSE_REPOSITORY_URL: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.repository') }}" DISCOURSE_REF: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.ref') }}" -DISCOURSE_LDAP_AUTH_PLUGIN_REPOSITORY: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.plugins.discourse-ldap-auth.repository') }}" -DISCOURSE_LDAP_AUTH_PLUGIN_REF: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.plugins.discourse-ldap-auth.ref') }}" +DISCOURSE_LDAP_AUTH_PLUGIN_REPOSITORY: "{{ lookup('config', application_id, 'addons.discourse-ldap-auth.config.repository') }}" +DISCOURSE_LDAP_AUTH_PLUGIN_REF: "{{ lookup('config', application_id, 'addons.discourse-ldap-auth.config.ref') }}" ## Database DISCOURSE_DATABASE: "{{ lookup('database', application_id ) }}" diff --git a/roles/web-app-erpnext/meta/services.yml b/roles/web-app-erpnext/meta/services.yml index daa58be0f2..2a48c1c3e6 100644 --- a/roles/web-app-erpnext/meta/services.yml +++ b/roles/web-app-erpnext/meta/services.yml @@ -1,49 +1,60 @@ --- # nocheck: seaweedfs-required โ€” object storage not implemented for this role sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: shared # nocheck: dynamic-flag # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests mariadb: + bond: 1 enabled: true shared: "{{ 'svc-db-mariadb' in group_names }}" # nocheck: shared # nocheck: dynamic-flag # nocheck: playwright-service-flag โ€” cache/queue/socketio buses, covered by role-local integration tests redis: + bond: 1 enabled: true shared: "{{ 'svc-db-redis' in group_names }}" # nocheck: playwright-service-flag โ€” self-provider erpnext: + bond: 1 enabled: true shared: true image: frappe/erpnext - version: "v16.23.1" + version: "v16.25.0" name: erpnext min_storage: 15GB ports: @@ -53,7 +64,7 @@ erpnext: run_after: - web-app-keycloak - web-app-mailu - lifecycle: alpha + lifecycle: beta cpus: "2.0" mem_reservation: 2g mem_limit: 4g diff --git a/roles/web-app-erpnext/meta/variants.yml b/roles/web-app-erpnext/meta/variants.yml index 49a99b8f70..235d95fe6c 100644 --- a/roles/web-app-erpnext/meta/variants.yml +++ b/roles/web-app-erpnext/meta/variants.yml @@ -85,7 +85,7 @@ enabled: false shared: false mariadb: - shared: false + shared: true sso: enabled: false shared: false @@ -93,4 +93,4 @@ enabled: false shared: false redis: - shared: false + shared: true diff --git a/roles/web-app-espocrm/meta/services.yml b/roles/web-app-espocrm/meta/services.yml index 0160955cc7..6a7373faa8 100644 --- a/roles/web-app-espocrm/meta/services.yml +++ b/roles/web-app-espocrm/meta/services.yml @@ -1,25 +1,32 @@ --- # nocheck: seaweedfs-required โ€” object storage not implemented for this role sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" mariadb: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-mariadb' in group_names }}" espocrm: @@ -53,11 +60,13 @@ websocket: mem_limit: 0.5g pids_limit: 384 css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag recaptcha: enabled: "{{ RECAPTCHA_ENABLED | bool }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-espocrm/tasks/main.yml b/roles/web-app-espocrm/tasks/main.yml index 7af16a2891..f15f0bf872 100644 --- a/roles/web-app-espocrm/tasks/main.yml +++ b/roles/web-app-espocrm/tasks/main.yml @@ -4,6 +4,8 @@ name: sys-stk-full vars: compose_handlers_flush: false + webserver_websocket_location: "/ws" + webserver_vhost_flavour: "ws_generic" - name: "Deploy '{{ ESPOCRM_ENTRYPOINT_SCRIPT_HOST_ABS }}'" copy: diff --git a/roles/web-app-espocrm/vars/main.yml b/roles/web-app-espocrm/vars/main.yml index a4e2e57f86..c093bbf9be 100644 --- a/roles/web-app-espocrm/vars/main.yml +++ b/roles/web-app-espocrm/vars/main.yml @@ -6,10 +6,7 @@ entity_name: "{{ application_id | get_entity_name }}" # Database # Webserver -webserver_websocket_location: "/ws" websocket_port: "{{ lookup('config', application_id, 'services.espocrm.ports.local.websocket') }}" -webserver_client_max_body_size: "100m" -webserver_vhost_flavour: "ws_generic" # Espocrm diff --git a/roles/web-app-fediwall/meta/services.yml b/roles/web-app-fediwall/meta/services.yml index 2a5341fb0e..f6c928748f 100644 --- a/roles/web-app-fediwall/meta/services.yml +++ b/roles/web-app-fediwall/meta/services.yml @@ -6,14 +6,17 @@ logout: shared: false # nocheck: dynamic-flag dashboard: + bond: 1 enabled: true shared: true # nocheck: dynamic-flag matomo: + bond: 1 enabled: true shared: true # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag @@ -29,12 +32,15 @@ sso: shared: false # nocheck: dynamic-flag prometheus: + bond: 1 enabled: true shared: true mastodon: + bond: 1 enabled: "{{ 'web-app-mastodon' in group_names }}" shared: "{{ 'web-app-mastodon' in group_names }}" friendica: + bond: 1 enabled: "{{ 'web-app-friendica' in group_names }}" shared: "{{ 'web-app-friendica' in group_names }}" fediwall: @@ -77,7 +83,7 @@ fediwall: ports: local: http: 8074 - lifecycle: alpha + lifecycle: beta min_storage: 1GB cpus: "0.2" mem_reservation: 128m diff --git a/roles/web-app-fider/meta/services.yml b/roles/web-app-fider/meta/services.yml index 6d5b741f33..064d72db7f 100644 --- a/roles/web-app-fider/meta/services.yml +++ b/roles/web-app-fider/meta/services.yml @@ -1,22 +1,28 @@ --- sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" fider: @@ -46,6 +52,7 @@ redis: enabled: false # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag @@ -53,8 +60,10 @@ css: javascript: enabled: false prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" diff --git a/roles/web-app-flowise/meta/services.yml b/roles/web-app-flowise/meta/services.yml index c0c90e6c88..9e39e18489 100644 --- a/roles/web-app-flowise/meta/services.yml +++ b/roles/web-app-flowise/meta/services.yml @@ -1,14 +1,18 @@ --- logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -44,7 +48,11 @@ qdrant: mem_limit: 4g pids_limit: 2048 min_storage: 1GB +# nocheck: playwright-service-flag: shared provider entity; the app itself is covered by this role's own Playwright spec flowise: + bond: 1 + enabled: true + shared: true backup: no_stop_required: false image: flowiseai/flowise @@ -72,10 +80,12 @@ redis: enabled: false # nocheck: playwright-service-flag โ€” AI backend, infrastructural ollama: + bond: 1 enabled: "{{ 'svc-ai-ollama' in group_names }}" shared: "{{ 'svc-ai-ollama' in group_names }}" # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag @@ -84,8 +94,10 @@ javascript: enabled: false shared: true email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-flowise/tasks/01_core.yml b/roles/web-app-flowise/tasks/01_core.yml new file mode 100644 index 0000000000..8cc2aa06a3 --- /dev/null +++ b/roles/web-app-flowise/tasks/01_core.yml @@ -0,0 +1,24 @@ +--- +- include_tasks: utils/once/flag.yml + +- name: "load docker and db for {{ application_id }}" + include_role: + name: sys-stk-backend + vars: + compose_handlers_flush: false + +- name: "create {{ FLOWISE_LITELLM_CONFIG_PATH_HOST }}" + template: + src: "litellm.config.yaml.j2" + dest: "{{ FLOWISE_LITELLM_CONFIG_PATH_HOST }}" + notify: compose up + +- name: "flush handlers of compose" + meta: flush_handlers + +- name: "Include role sys-stk-front-proxy for '{{ application_id }}'" + include_role: + name: sys-stk-front-proxy + vars: + domain: "{{ lookup('domain',application_id) }}" + http_port: "{{ lookup('config', application_id, 'services.flowise.ports.local.http') }}" diff --git a/roles/web-app-flowise/tasks/main.yml b/roles/web-app-flowise/tasks/main.yml index 7fb50de202..a3425c1e43 100644 --- a/roles/web-app-flowise/tasks/main.yml +++ b/roles/web-app-flowise/tasks/main.yml @@ -1,22 +1,6 @@ --- -- name: "load docker and db for {{ application_id }}" - include_role: - name: sys-stk-backend +- name: "Execute Role (once)" + include_tasks: 01_core.yml + when: run_once_web_app_flowise is not defined vars: - compose_handlers_flush: false - -- name: "create {{ FLOWISE_LITELLM_CONFIG_PATH_HOST }}" - template: - src: "litellm.config.yaml.j2" - dest: "{{ FLOWISE_LITELLM_CONFIG_PATH_HOST }}" - notify: compose up - -- name: "flush handlers of compose" - meta: flush_handlers - -- name: "Include role sys-stk-front-proxy for '{{ application_id }}'" - include_role: - name: sys-stk-front-proxy - vars: - domain: "{{ lookup('domain',application_id) }}" - http_port: "{{ lookup('config', application_id, 'services.flowise.ports.local.http') }}" + application_id: "web-app-flowise" diff --git a/roles/web-app-friendica/README.md b/roles/web-app-friendica/README.md index af69ef21a3..444dd44018 100644 --- a/roles/web-app-friendica/README.md +++ b/roles/web-app-friendica/README.md @@ -18,6 +18,19 @@ For detailed administration procedures, please refer to the [Administration.md]( - **Configuration Debugging:** Quickly inspect environment variables, volume data, and configuration files to troubleshoot issues. - **Autoinstall Capability:** Automate initial installation steps to rapidly deploy a working Friendica instance. +## Addons + +Role-level extensions are declared in [`meta/addons/`](./meta/addons/) +(unified addon contract, requirement 026): + +| Addon | Mechanism | Default state | Bridges | +|-------|-----------|---------------|---------| +| `ldapauth` | `addon` | enabled whenever the `ldap` service is present (`svc-db-openldap` co-deployed) | `ldap` โ†’ `svc-db-openldap` | + +`ldapauth` is the only path that materialises a `friendica.user` row, so its enablement derives directly from the `ldap` service flag. +The oauth2-proxy `sso` gate in front of the vhost is a front-door auth gate, not an addon bridge, and stays in [`meta/services.yml`](./meta/services.yml). +The LDAP login path is covered by the existing LDAP Playwright spec (requirement 018), so no addon-specific spec is added. + ## Further Resources - [Friendica Docker Hub](https://hub.docker.com/_/friendica) diff --git a/roles/web-app-friendica/files/playwright/addons/ldapauth.spec.js b/roles/web-app-friendica/files/playwright/addons/ldapauth.spec.js new file mode 100644 index 0000000000..74e9f16672 --- /dev/null +++ b/roles/web-app-friendica/files/playwright/addons/ldapauth.spec.js @@ -0,0 +1,106 @@ +const { test, expect } = require("@playwright/test"); +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const { skipUnlessServiceEnabled } = require("../service-gating"); +const shared = require("../_shared"); + +// ldapauth is Friendica's LDAP authentication addon. It has no dedicated +// in-app page; its coupling to svc-db-openldap is observable as: (a) a valid +// LDAP credential binds against the openldap partner and establishes a +// Friendica session, and (b) an INVALID credential is REJECTED by that same +// bind โ€” proof the addon actually delegates auth to LDAP rather than the page +// merely re-rendering. We reuse the role's LDAP login idiom (the exact +// ldapauth bind path) and gate behind both the addon flag and the ldap +// service. Both assertions FAIL if the integration is not wired. +test("addon ldapauth: a valid LDAP credential binds against openldap and a bogus one is rejected", async ({ browser }) => { + skipUnlessAddonEnabled("ldapauth"); + skipUnlessServiceEnabled("ldap"); + + const context = await browser.newContext({ ignoreHTTPSErrors: true }); + const page = await context.newPage(); + + try { + const baseUrl = shared.trimmedBaseUrl(); + const login = shared.pickLoginPath(); + + // 1) Positive path: the real LDAP credential must establish a session. + await login(page, shared.env.adminUsername, shared.env.adminPassword); + await page.goto(`${baseUrl}/network`, { waitUntil: "domcontentloaded" }).catch(() => {}); + + const authenticatedSurface = page + .locator("#topbar-first, #navbar-apps-menu, a[href*='/logout']") + .first(); + await expect( + authenticatedSurface, + "Expected an authenticated Friendica surface after the ldapauth-backed login" + ).toBeVisible({ timeout: 60_000 }); + + await expect( + page.locator("input[name='password']"), + "Expected to be past the login form once ldapauth accepted the LDAP credential" + ).toHaveCount(0); + + // 2) When the admin user can reach /admin/addons, the addon list must show + // ldapauth as installed/active โ€” direct proof the addon is enabled and the + // hooks are registered. Admin reachability depends on the admin-email match, + // so we only assert when the panel is reachable; otherwise the bind-based + // checks above and below carry the coupling proof. + await page.goto(`${baseUrl}/admin/addons`, { waitUntil: "domcontentloaded" }).catch(() => {}); + const onAdminAddons = /\/admin\/addons/.test(page.url()); + const hasLoginForm = await page + .locator("input[name='password']") + .first() + .isVisible({ timeout: 2_000 }) + .catch(() => false); + if (onAdminAddons && !hasLoginForm) { + await expect( + page.locator("a[href*='admin/addons/ldapauth'], :text('ldapauth')").first(), + "Expected ldapauth to appear in Friendica's admin addon list when the admin panel is reachable" + ).toBeVisible({ timeout: 15_000 }); + } + + await shared.friendicaLogout(page); + } finally { + await page.close().catch(() => {}); + await context.close().catch(() => {}); + } + + // 3) Negative path on a fresh context: a credential with the right username + // but a deliberately WRONG password must NOT authenticate. ldapauth binds + // that password against openldap and the bind fails, so no Friendica session + // is created. If auth were stubbed/bypassed (integration not wired), the + // session would establish and this assertion would fail โ€” which is exactly + // the regression we want to catch. + const negContext = await browser.newContext({ ignoreHTTPSErrors: true }); + const negPage = await negContext.newPage(); + try { + const baseUrl = shared.trimmedBaseUrl(); + await negPage.goto(`${baseUrl}/login`, { waitUntil: "domcontentloaded" }).catch(() => {}); + + const hasForm = await negPage + .locator("input[name='password']") + .first() + .isVisible({ timeout: 30_000 }) + .catch(() => false); + + // Only meaningful where Friendica's own login form is directly reachable + // (the ldap-only matrix variant). With oauth2-proxy in front, the bogus + // password is rejected at Keycloak before ldapauth is ever consulted, so + // the negative assertion would test Keycloak, not this addon โ€” skip it. + if (hasForm && !shared.isServiceEnabled("sso")) { + await shared.loginViaFriendicaForm( + negPage, + shared.env.adminUsername, + `wrong-${Date.now()}-${shared.env.adminPassword}` + ); + await negPage.goto(`${baseUrl}/network`, { waitUntil: "domcontentloaded" }).catch(() => {}); + + await expect( + negPage.locator("#topbar-first, #navbar-apps-menu, a[href*='/logout']").first(), + "Expected NO authenticated session when ldapauth rejects a bogus password against openldap" + ).toHaveCount(0); + } + } finally { + await negPage.close().catch(() => {}); + await negContext.close().catch(() => {}); + } +}); diff --git a/roles/web-app-friendica/meta/addons/ldapauth.yml b/roles/web-app-friendica/meta/addons/ldapauth.yml new file mode 100644 index 0000000000..0158a33cc7 --- /dev/null +++ b/roles/web-app-friendica/meta/addons/ldapauth.yml @@ -0,0 +1,11 @@ +--- +enabled: "{{ lookup('config', 'web-app-friendica', 'services.ldap.enabled') | bool }}" +required: false +mechanism: addon +source: upstream +bridges: + - ldap +update: + monitored: true + catalog: friendica-addons + upstream_id: ldapauth diff --git a/roles/web-app-friendica/meta/services.yml b/roles/web-app-friendica/meta/services.yml index bfece33b21..039e2489a9 100644 --- a/roles/web-app-friendica/meta/services.yml +++ b/roles/web-app-friendica/meta/services.yml @@ -1,18 +1,13 @@ --- -# nocheck: seaweedfs-required โ€” object storage not implemented for this role -# nocheck: playwright-service-flag โ€” self-provider +# nocheck: seaweedfs-required, object storage not implemented for this role +# nocheck: playwright-service-flag, self-provider friendica: + bond: 1 shared: true enabled: true image: friendica version: "2026.05" min_storage: 10GB - addons: - # SSO is owned by the oauth2-proxy gate in front of the friendica vhost - # (services.sso.enabled=keycloak below). `ldapauth` stays enabled - # because friendica has no header-trusted autocreate mechanism โ€” the - # ldap bind is the only path that materialises a friendica.user row. - ldapauth: {} ports: local: sso: 16501 @@ -26,12 +21,15 @@ friendica: mem_limit: 2g pids_limit: 1024 logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -44,10 +42,10 @@ sso: # network / settings views, the admin panel) all force the Keycloak # round-trip. A blacklist that opens `/` and only gates `/login` # strips the POST body on the 401 redirect and breaks ldapauth. - # Everything Friendica considers naturally public โ€” federation API, + # Everything Friendica considers naturally public (federation API, # ActivityPub / NodeInfo / WebFinger discovery, public profile and # post pages, the community feed, media, search, the theme assets - # and the static instance-info pages โ€” is whitelisted here so + # and the static instance-info pages) is whitelisted here so # anonymous visitors (and the Fediwall SPA, foreign instances, and # crawlers) can reach those surfaces without being bounced through # Keycloak. @@ -82,22 +80,28 @@ sso: - /privacy - /tos dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" -# nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests +# nocheck: playwright-service-flag, DB engine, covered by role-local integration tests mariadb: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-mariadb' in group_names }}" -# nocheck: playwright-service-flag โ€” verified by web-svc-css provider spec +# nocheck: playwright-service-flag, verified by web-svc-css provider spec css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-friendica/tasks/04_addons.yml b/roles/web-app-friendica/tasks/04_addons.yml index 04cdbb3513..1350972558 100644 --- a/roles/web-app-friendica/tasks/04_addons.yml +++ b/roles/web-app-friendica/tasks/04_addons.yml @@ -2,18 +2,14 @@ - name: flush handlers to ensure that friendica is up before friendica addon configuration meta: flush_handlers -- name: Build friendica_addons based on features +- name: Build friendica_addons from meta/addons/ declarations set_fact: friendica_addons: >- {{ friendica_addons | default([]) + [{ 'name': item.key, - 'enabled': ( - FRIENDICA_LDAP_ENABLED - if item.key == 'ldapauth' - else (item.value.enabled if item.value is mapping and 'enabled' in item.value else False) - ) + 'enabled': (item.value.enabled | bool) }] }} loop: "{{ FRIENDICA_ADDONS | dict2items }}" @@ -117,3 +113,10 @@ stdin: "{{ lookup('file', role_path + '/files/php/verify_addon_hooks.php') }}" changed_when: false when: FRIENDICA_LDAP_ENABLED | bool + +- name: "Run per-addon integration hooks for enabled addons" + include_tasks: "addons/{{ item.name }}.yml" + loop: "{{ friendica_addons | selectattr('enabled') | list }}" + loop_control: + label: "{{ item.name }}" + when: (role_path ~ '/tasks/addons/' ~ item.name ~ '.yml') is exists diff --git a/roles/web-app-friendica/tasks/addons/ldapauth.yml b/roles/web-app-friendica/tasks/addons/ldapauth.yml new file mode 100644 index 0000000000..672ebd18a5 --- /dev/null +++ b/roles/web-app-friendica/tasks/addons/ldapauth.yml @@ -0,0 +1,2 @@ +--- +# No exception logic required: the generic config SPOT (meta/addons/.yml plugin_configuration + tasks/plugin) fully wires this integration; coupling is verified by the Playwright spec. diff --git a/roles/web-app-friendica/templates/playwright.env.j2 b/roles/web-app-friendica/templates/playwright.env.j2 index 4c805b9df1..b9f3bc82cb 100644 --- a/roles/web-app-friendica/templates/playwright.env.j2 +++ b/roles/web-app-friendica/templates/playwright.env.j2 @@ -36,3 +36,5 @@ PERSONA_BIBER_BLOCKED=true SSO_SERVICE_ENABLED={{ lookup('config', application_id, 'services.sso.enabled') | string | lower }} +{{ lookup('addon_env_flags', application_id) }} + diff --git a/roles/web-app-friendica/vars/main.yml b/roles/web-app-friendica/vars/main.yml index 8dcabd4f8b..3a5db94901 100644 --- a/roles/web-app-friendica/vars/main.yml +++ b/roles/web-app-friendica/vars/main.yml @@ -14,7 +14,7 @@ FRIENDICA_APPLICATION_DIR: "/var/www/html" FRIENDICA_CONFIG_DIR: "{{ [ FRIENDICA_APPLICATION_DIR, 'config' ] | path_join }}" FRIENDICA_CONFIG_FILE: "{{ [ FRIENDICA_CONFIG_DIR, 'local.config.php' ] | path_join }}" FRIENDICA_USER: "www-data" -FRIENDICA_ADDONS: "{{ lookup('config', application_id, 'services.friendica.addons') }}" +FRIENDICA_ADDONS: "{{ lookup('config', application_id, 'addons') }}" # IAM FRIENDICA_OIDC_ENABLED: "{{ lookup('config', application_id, 'services.sso.enabled') }}" diff --git a/roles/web-app-funkwhale/meta/services.yml b/roles/web-app-funkwhale/meta/services.yml index 0fbf564fc3..5171f3b6a9 100644 --- a/roles/web-app-funkwhale/meta/services.yml +++ b/roles/web-app-funkwhale/meta/services.yml @@ -33,18 +33,23 @@ typesense: mem_limit: 256m pids_limit: 256 logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: shared @@ -53,7 +58,12 @@ redis: backup: disabled: true enabled: true + cpus: "0.5" + mem_reservation: 128m + mem_limit: 512m + pids_limit: 512 postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" # nocheck: playwright-service-flag โ€” object store, no Playwright surface @@ -62,9 +72,11 @@ minio: shared: false # nocheck: dynamic-flag # nocheck: playwright-service-flag โ€” object store, no Playwright surface seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -76,8 +88,10 @@ sso: blacklist: - /login css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-funkwhale/meta/variants.yml b/roles/web-app-funkwhale/meta/variants.yml index 3c0ae83f39..b11a9ebeb7 100644 --- a/roles/web-app-funkwhale/meta/variants.yml +++ b/roles/web-app-funkwhale/meta/variants.yml @@ -87,7 +87,7 @@ enabled: false shared: false postgres: - shared: false + shared: true prometheus: enabled: false shared: false diff --git a/roles/web-app-funkwhale/tasks/main.yml b/roles/web-app-funkwhale/tasks/main.yml index 2d0d5b0710..5f04244316 100644 --- a/roles/web-app-funkwhale/tasks/main.yml +++ b/roles/web-app-funkwhale/tasks/main.yml @@ -2,3 +2,5 @@ - name: "load docker, db and proxy for {{ application_id }}" include_role: name: sys-stk-full + vars: + webserver_extra_configuration: "client_max_body_size 512M;" diff --git a/roles/web-app-funkwhale/vars/main.yml b/roles/web-app-funkwhale/vars/main.yml index 5df619e62b..24615b3f79 100644 --- a/roles/web-app-funkwhale/vars/main.yml +++ b/roles/web-app-funkwhale/vars/main.yml @@ -1,6 +1,5 @@ --- application_id: "web-app-funkwhale" -webserver_extra_configuration: "client_max_body_size 512M;" FUNKWHALE_DATA_ROOT: "/var/lib/funkwhale/" FUNKWHALE_MEDIA_DIR: "{{ [ FUNKWHALE_DATA_ROOT, 'media' ] | path_join }}" diff --git a/roles/web-app-fusiondirectory/meta/services.yml b/roles/web-app-fusiondirectory/meta/services.yml index 217322559a..12286264a5 100644 --- a/roles/web-app-fusiondirectory/meta/services.yml +++ b/roles/web-app-fusiondirectory/meta/services.yml @@ -1,23 +1,30 @@ --- logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" ldap: + bond: 1 enabled: true # nocheck: dynamic-flag shared: true # nocheck: dynamic-flag โ€” LDAP IS the storage backend, never local sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -48,5 +55,6 @@ fusiondirectory: pids_limit: 512 # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" diff --git a/roles/web-app-gitea/meta/services.yml b/roles/web-app-gitea/meta/services.yml index bf18adc162..eb9f679e79 100644 --- a/roles/web-app-gitea/meta/services.yml +++ b/roles/web-app-gitea/meta/services.yml @@ -1,12 +1,15 @@ --- # nocheck: seaweedfs-playwright โ€” Gitea uses S3 for storage, but its UI login is not automatable here (oauth2-proxy + OIDC account-link / native login both fail; personas are blocked), so a login-dependent storage check cannot run logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -18,20 +21,25 @@ sso: blacklist: - /user/login dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests mariadb: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-mariadb' in group_names }}" # nocheck: playwright-service-flag โ€” self-provider gitea: + bond: 1 enabled: true shared: true provides: gitea @@ -80,13 +88,16 @@ minio: shared: false # nocheck: dynamic-flag # nocheck: playwright-service-flag โ€” object store; storage check needs a UI login, but Gitea's personas are blocked (PERSONA_ADMINISTRATOR_BLOCKED/PERSONA_BIBER_BLOCKED), so it has no automatable Playwright login surface seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" native_metrics: diff --git a/roles/web-app-gitlab/meta/services.yml b/roles/web-app-gitlab/meta/services.yml index c4121eca7a..85946481af 100644 --- a/roles/web-app-gitlab/meta/services.yml +++ b/roles/web-app-gitlab/meta/services.yml @@ -1,19 +1,24 @@ --- # nocheck: seaweedfs-playwright โ€” GitLab ships no templates/playwright.env.j2 and object storage is server-side consolidated storage with no isolated upload surface sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: shared @@ -22,12 +27,20 @@ redis: backup: disabled: true enabled: true + cpus: "0.5" + mem_reservation: 128m + mem_limit: 512m + pids_limit: 512 postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" gitlab: + bond: 1 + enabled: true + shared: true image: gitlab/gitlab-ee - version: 19.1.0-ee.0 + version: 19.1.1-ee.0 name: gitlab min_storage: 30GB cpus: "4.0" @@ -45,11 +58,14 @@ gitlab: - web-app-mailu lifecycle: beta seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-gitlab/tasks/01_core.yml b/roles/web-app-gitlab/tasks/01_core.yml new file mode 100644 index 0000000000..6ddfd26222 --- /dev/null +++ b/roles/web-app-gitlab/tasks/01_core.yml @@ -0,0 +1,6 @@ +--- +- include_tasks: utils/once/flag.yml + +- name: "load docker, db and proxy for {{ application_id }}" + include_role: + name: sys-stk-full diff --git a/roles/web-app-gitlab/tasks/main.yml b/roles/web-app-gitlab/tasks/main.yml index 2d0d5b0710..6751d8396a 100644 --- a/roles/web-app-gitlab/tasks/main.yml +++ b/roles/web-app-gitlab/tasks/main.yml @@ -1,4 +1,6 @@ --- -- name: "load docker, db and proxy for {{ application_id }}" - include_role: - name: sys-stk-full +- name: "Execute Role (once)" + include_tasks: 01_core.yml + when: run_once_web_app_gitlab is not defined + vars: + application_id: "web-app-gitlab" diff --git a/roles/web-app-hugo/meta/services.yml b/roles/web-app-hugo/meta/services.yml index c0c0e055c7..50874d5750 100644 --- a/roles/web-app-hugo/meta/services.yml +++ b/roles/web-app-hugo/meta/services.yml @@ -11,13 +11,16 @@ sso: enabled: false shared: false dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" hugo: @@ -32,7 +35,7 @@ hugo: ports: local: http: 8008 - lifecycle: alpha + lifecycle: beta min_storage: 1GB cpus: "0.2" mem_reservation: 128m @@ -44,5 +47,6 @@ email: enabled: false shared: false prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-jenkins/meta/services.yml b/roles/web-app-jenkins/meta/services.yml index fa411a096e..b64facb0fc 100644 --- a/roles/web-app-jenkins/meta/services.yml +++ b/roles/web-app-jenkins/meta/services.yml @@ -1,21 +1,27 @@ --- logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" jenkins: @@ -36,5 +42,6 @@ jenkins: pids_limit: 1024 # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" diff --git a/roles/web-app-jira/meta/services.yml b/roles/web-app-jira/meta/services.yml index 54c2f4afdd..ce1e3335c2 100644 --- a/roles/web-app-jira/meta/services.yml +++ b/roles/web-app-jira/meta/services.yml @@ -1,6 +1,7 @@ --- # nocheck: seaweedfs-required โ€” object storage not implemented for this role logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" # nocheck: dynamic-flag @@ -10,9 +11,11 @@ sso: enabled: false shared: false dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" # nocheck: dynamic-flag @@ -21,6 +24,7 @@ email: enabled: false shared: false postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" jira: @@ -39,8 +43,10 @@ jira: - web-app-matomo lifecycle: unsupported css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-jitsi/meta/services.yml b/roles/web-app-jitsi/meta/services.yml index 13ba094e93..43e0955765 100644 --- a/roles/web-app-jitsi/meta/services.yml +++ b/roles/web-app-jitsi/meta/services.yml @@ -1,5 +1,6 @@ --- sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -8,26 +9,33 @@ sso: host: application port: 80 ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" # nocheck: playwright-service-flag โ€” email not used by Jitsi room flow email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag โ€” gated on services.sso.enabled, not on group_names @@ -36,6 +44,7 @@ css: # Jitsi's React SPA (which has no native sign-out control) still exposes a click target # that survives the persona contract's no-direct-URL logout rule. javascript: + bond: 1 enabled: "{{ lookup('config', 'web-app-jitsi', 'services.sso.enabled') | bool }}" shared: "{{ lookup('config', 'web-app-jitsi', 'services.sso.enabled') | bool }}" jitsi: @@ -50,7 +59,7 @@ jitsi: media: 10000 run_after: - web-app-keycloak - lifecycle: alpha + lifecycle: beta cpus: "2.0" mem_reservation: 1g mem_limit: 3g diff --git a/roles/web-app-joomla/README.md b/roles/web-app-joomla/README.md index 405df944f8..fe4b669851 100644 --- a/roles/web-app-joomla/README.md +++ b/roles/web-app-joomla/README.md @@ -10,6 +10,18 @@ This role deploys Joomla on Docker Compose. It builds a custom Joomla image that For the OIDC plugin source and its environment contract see [README.md](./files/joomla-oidc-plugin/README.md). +## Addons + +Role-level extensions are declared in [`meta/addons/`](./meta/addons/) +(unified addon contract, requirement 026): + +| Addon | Mechanism | Default state | Bridges | +|-------|-----------|---------------|---------| +| `plg_system_keycloak` | `plugin` | enabled whenever the `sso` service is present (`web-app-keycloak` co-deployed) | `sso` โ†’ `web-app-keycloak` | + +`plg_system_keycloak` is built imperatively inside the Joomla container and speaks OIDC against Keycloak. +Its enablement derives directly from the `sso` service flag, and its OIDC runtime contract (issuer, client id/secret, redirect/end-session URLs, fallback toggle, and RBAC group paths) lives under the addon's `config:` block, which [`templates/env.j2`](./templates/env.j2) renders into the `JOOMLA_OIDC_*` plugin environment. + ## Features - **Containerized deployment:** Run Joomla through Docker Compose with the role-specific custom image. @@ -25,11 +37,11 @@ See [README.md](./files/joomla-oidc-plugin/README.md) for the OIDC plugin's mani ## End-to-end tests -The Playwright spec at [files/playwright/playwright.spec.js](./files/playwright/playwright.spec.js) currently exercises the **administrator** path only: Keycloak SSO into the admin backend plus the local form-login emergency hatch (`?fallback=local`). The non-admin RBAC path via the canonical `biber` user is not yet covered. +The Playwright spec at [`playwright.spec.js`](./files/playwright/playwright.spec.js) currently exercises the **administrator** path only: Keycloak SSO into the admin backend plus the local form-login emergency hatch (`?fallback=local`). The non-admin RBAC path via the canonical `biber` user is not yet covered. Until the biber path is added: -- the role's [templates/playwright.env.j2](./templates/playwright.env.j2) MUST NOT carry stale `BIBER_USERNAME` / `BIBER_PASSWORD` keys (the lint at [tests/lint/ansible/roles/web-app/playwright/test_env_keys_used.py](../../tests/lint/ansible/roles/web-app/playwright/test_env_keys_used.py) enforces this); +- the role's [`playwright.env.j2`](./templates/playwright.env.j2) MUST NOT carry stale `BIBER_USERNAME` / `BIBER_PASSWORD` keys (the lint at [`test_env_keys_used.py`](../../tests/lint/ansible/roles/web-app/playwright/test_env_keys_used.py) enforces this); - the admin scenarios stay gated on `oidc` (and on `ldap` for the LDAP-variant scenarios), so a deploy with `disable=oidc` reports the SSO scenario as `skipped`, never `failed`. ## Further Resources diff --git a/roles/web-app-joomla/files/joomla-oidc-plugin/plg_system_keycloak/src/Extension/Keycloak.php b/roles/web-app-joomla/files/joomla-oidc-plugin/plg_system_keycloak/src/Extension/Keycloak.php index 233eb16700..1cd7442305 100644 --- a/roles/web-app-joomla/files/joomla-oidc-plugin/plg_system_keycloak/src/Extension/Keycloak.php +++ b/roles/web-app-joomla/files/joomla-oidc-plugin/plg_system_keycloak/src/Extension/Keycloak.php @@ -174,12 +174,40 @@ class Keycloak extends CMSPlugin $client->addScope(['openid', 'profile', 'email', 'groups']); $client->setVerifyHost(true); $client->setVerifyPeer(true); - if (file_exists('/etc/ssl/certs/ca-certificates.crt')) { - $client->setCertPath('/etc/ssl/certs/ca-certificates.crt'); + $caPath = $this->resolveCaBundlePath(); + if ($caPath !== '') { + $client->setCertPath($caPath); } return $client; } + /** + * Resolve the CA bundle for the server-side OIDC discovery cURL. + * + * The sys-svc-compose-ca override reliably bind-mounts the Infinito + * root CA and exports CURL_CA_BUNDLE/CA_TRUST_CERT/SSL_CERT_FILE to + * point at it, even when the CA-trust entrypoint wrapper was not + * applied (e.g. locally-built image absent at inject time). Prefer + * those over the system bundle, which only carries the Infinito CA + * once update-ca-certificates has run inside the container. + */ + private function resolveCaBundlePath(): string + { + $candidates = [ + getenv('CURL_CA_BUNDLE') ?: '', + getenv('CA_TRUST_CERT') ?: '', + getenv('SSL_CERT_FILE') ?: '', + '/etc/ssl/certs/ca-certificates.crt', + ]; + foreach ($candidates as $candidate) { + $candidate = (string) $candidate; + if ($candidate !== '' && is_file($candidate)) { + return $candidate; + } + } + return ''; + } + private function handleCallback(): void { $oidc = $this->buildOidcClient(); diff --git a/roles/web-app-joomla/files/playwright/addons/plg_system_keycloak.spec.js b/roles/web-app-joomla/files/playwright/addons/plg_system_keycloak.spec.js new file mode 100644 index 0000000000..f099e68f4f --- /dev/null +++ b/roles/web-app-joomla/files/playwright/addons/plg_system_keycloak.spec.js @@ -0,0 +1,99 @@ +const { test, expect } = require("@playwright/test"); +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const { skipUnlessServiceEnabled } = require("../service-gating"); +const { + decodeDotenvQuotedValue, + normalizeBaseUrl, + performKeycloakLoginForm, +} = require("../personas"); + +// plg_system_keycloak is Joomla's native OIDC SSO plugin (in-role plugin, +// configured purely via JOOMLA_OIDC_* env vars injected by templates/env.j2; +// installed + enabled by tasks/07_oidc_plugin.yml). It has no in-app settings +// panel, so its observable coupling surface is the live OIDC handshake: +// 1. The plugin's onAfterInitialise guard redirects an unauthenticated +// visitor to the *configured* Keycloak issuer (proves issuer/client_id +// env wiring) โ€” NOT to a generic login page. +// 2. After a real Keycloak credential exchange the browser lands back on +// Joomla in an AUTHENTICATED session (proves the redirect_uri/client_secret +// callback completed and the plugin provisioned/logged in the user). +// 3. Re-requesting the guarded root then no longer bounces to Keycloak +// (proves the session is established, not a redirect loop). +// This asserts the integration end to end and FAILS if the plugin is not +// installed, not enabled, or the OIDC client env is missing/mismatched. +const joomlaBaseUrl = normalizeBaseUrl(process.env.JOOMLA_BASE_URL || ""); +const oidcIssuerUrl = normalizeBaseUrl(process.env.OIDC_ISSUER_URL || ""); +const adminUsername = decodeDotenvQuotedValue(process.env.ADMIN_USERNAME); +const adminPassword = decodeDotenvQuotedValue(process.env.ADMIN_PASSWORD); + +test.use({ ignoreHTTPSErrors: true }); + +test("addon plg_system_keycloak: OIDC handshake couples Joomla to the configured Keycloak and establishes an authenticated session", async ({ page }) => { + skipUnlessAddonEnabled("plg_system_keycloak"); + skipUnlessServiceEnabled("sso"); + + expect(joomlaBaseUrl, "JOOMLA_BASE_URL must be set").toBeTruthy(); + expect(adminUsername, "ADMIN_USERNAME must be set").toBeTruthy(); + expect(adminPassword, "ADMIN_PASSWORD must be set").toBeTruthy(); + expect(oidcIssuerUrl, "OIDC_ISSUER_URL must be set when OIDC is enabled").toBeTruthy(); + + await page.context().clearCookies(); + + const expectedOidcAuthUrl = `${oidcIssuerUrl}/protocol/openid-connect/auth`; + const expectedJoomlaBaseUrl = joomlaBaseUrl.replace(/\/$/, ""); + + // Step 1: the guard MUST redirect a guest to the CONFIGURED Keycloak issuer. + await page.goto(`${expectedJoomlaBaseUrl}/`); + + await expect + .poll(() => page.url(), { + timeout: 60_000, + message: `expected the plg_system_keycloak guard to redirect to the configured Keycloak OIDC auth endpoint (${expectedOidcAuthUrl})`, + }) + .toContain(expectedOidcAuthUrl); + + // The redirect must carry THIS Joomla's OIDC client_id, proving the + // env-injected client (not some default) is what the plugin handed to + // Keycloak. Keycloak echoes client_id in the auth URL. + const authUrl = new URL(page.url()); + expect( + authUrl.searchParams.get("client_id"), + "Expected the Keycloak auth redirect to carry an OIDC client_id from the injected JOOMLA_OIDC_CLIENT_ID", + ).toBeTruthy(); + expect( + authUrl.searchParams.get("redirect_uri") || "", + "Expected the OIDC redirect_uri to point back at this Joomla instance (proves JOOMLA_OIDC_REDIRECT_URI wiring)", + ).toContain(new URL(expectedJoomlaBaseUrl).host); + + // Step 2: real Keycloak credential exchange. + await performKeycloakLoginForm(page, adminUsername, adminPassword); + + await expect + .poll(() => page.url(), { + timeout: 60_000, + message: `expected the OIDC callback to land back on Joomla at ${expectedJoomlaBaseUrl}`, + }) + .toContain(expectedJoomlaBaseUrl); + + // Step 3: the callback must have established an authenticated session. + // Re-request the guarded root: a logged-in session is NOT bounced back to + // the Keycloak auth endpoint (the guard only redirects guests). A bounce + // here means the callback failed to provision/log in the user โ€” i.e. the + // integration is not actually coupled. + await page.goto(`${expectedJoomlaBaseUrl}/`); + await page.waitForLoadState("domcontentloaded", { timeout: 60_000 }).catch(() => {}); + + expect( + page.url(), + "Expected an authenticated Joomla session after the OIDC callback: re-requesting the guarded root must NOT bounce back to the Keycloak auth endpoint", + ).not.toContain(expectedOidcAuthUrl); + expect( + page.url(), + "Expected to remain on this Joomla instance after the OIDC handshake", + ).toContain(expectedJoomlaBaseUrl); + + await expect( + page.locator("body"), + "Expected the authenticated Joomla front-end to render after the plg_system_keycloak OIDC handshake", + ).toBeVisible({ timeout: 60_000 }); +}); diff --git a/roles/web-app-joomla/meta/addons/plg_system_keycloak.yml b/roles/web-app-joomla/meta/addons/plg_system_keycloak.yml new file mode 100644 index 0000000000..8eeec78887 --- /dev/null +++ b/roles/web-app-joomla/meta/addons/plg_system_keycloak.yml @@ -0,0 +1,18 @@ +--- +enabled: "{{ lookup('config', 'web-app-joomla', 'services.sso.enabled') | bool }}" +required: false +mechanism: plugin +source: built +version: "1.0.0" +bridges: + - sso +config: + issuer_url: "{{ OIDC.CLIENT.ISSUER_URL }}" + client_id: "{{ OIDC.CLIENT.ID }}" + client_secret: "{{ OIDC.CLIENT.SECRET }}" + redirect_uri: "{{ lookup('tls', 'web-app-joomla', 'url.base') }}/index.php?option=keycloak&task=callback" + end_session_url: "{{ OIDC.CLIENT.LOGOUT_URL }}" + fallback_enabled: "{{ lookup('config', 'web-app-joomla', 'services.sso.oidc.fallback_enabled') | string | lower }}" + group_admin: "/{{ lookup('rbac_group_path', application_id='web-app-joomla', role='administrator') }}" + group_editor: "/{{ lookup('rbac_group_path', application_id='web-app-joomla', role='editor') }}" + group_user: "/roles/web-app-joomla" diff --git a/roles/web-app-joomla/meta/services.yml b/roles/web-app-joomla/meta/services.yml index 71f20eade1..f342f8488c 100644 --- a/roles/web-app-joomla/meta/services.yml +++ b/roles/web-app-joomla/meta/services.yml @@ -1,6 +1,7 @@ --- # nocheck: seaweedfs-required โ€” object storage not implemented for this role sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc @@ -16,19 +17,24 @@ ldap: enabled: false shared: false logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests mariadb: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-mariadb' in group_names }}" joomla: @@ -55,8 +61,10 @@ joomla: pids_limit: 512 # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-joomla/tasks/addons/plg_system_keycloak.yml b/roles/web-app-joomla/tasks/addons/plg_system_keycloak.yml new file mode 100644 index 0000000000..672ebd18a5 --- /dev/null +++ b/roles/web-app-joomla/tasks/addons/plg_system_keycloak.yml @@ -0,0 +1,2 @@ +--- +# No exception logic required: the generic config SPOT (meta/addons/.yml plugin_configuration + tasks/plugin) fully wires this integration; coupling is verified by the Playwright spec. diff --git a/roles/web-app-joomla/tasks/main.yml b/roles/web-app-joomla/tasks/main.yml index 853e9fe886..3a59ae616a 100644 --- a/roles/web-app-joomla/tasks/main.yml +++ b/roles/web-app-joomla/tasks/main.yml @@ -7,6 +7,7 @@ loop_var: domain vars: http_port: "{{ lookup('config', application_id, 'services.joomla.ports.local.http') }}" + webserver_client_max_body_size: "{{ JOOMLA_POST_MAX_SIZE }}" - name: "load docker and db for {{ application_id }}" include_role: @@ -46,4 +47,4 @@ - name: "Install plg_system_keycloak (native OIDC)" include_tasks: 07_oidc_plugin.yml - when: lookup('config', application_id, 'services.sso.enabled') | bool + when: lookup('config', application_id, 'addons.plg_system_keycloak.enabled') | bool diff --git a/roles/web-app-joomla/templates/env.j2 b/roles/web-app-joomla/templates/env.j2 index 9bc4f7c07c..a22a817e04 100644 --- a/roles/web-app-joomla/templates/env.j2 +++ b/roles/web-app-joomla/templates/env.j2 @@ -13,18 +13,19 @@ JOOMLA_DB_NAME={{ lookup('database', application_id, 'name') }} JOOMLA_DB_TYPE={{ JOOMLA_DB_CONNECTOR }} {% endif %} -{% if lookup('config', application_id, 'services.sso.enabled') | bool %} +{% if lookup('config', application_id, 'addons.plg_system_keycloak.enabled') | bool %} +{% set JOOMLA_OIDC = lookup('config', application_id, 'addons.plg_system_keycloak.config') %} # OIDC SSO via the in-role plg_system_keycloak plugin. # Modus 3: Force-Frontend, Local-Backup-Backend. The fallback hatch at # /administrator?fallback=local can be disabled by flipping # services.sso.oidc.fallback_enabled to false in this role's meta/services.yml. -JOOMLA_OIDC_ISSUER_URL={{ OIDC.CLIENT.ISSUER_URL }} -JOOMLA_OIDC_CLIENT_ID={{ OIDC.CLIENT.ID }} -JOOMLA_OIDC_CLIENT_SECRET={{ OIDC.CLIENT.SECRET | dotenv_quote }} -JOOMLA_OIDC_REDIRECT_URI={{ lookup('tls', application_id, 'url.base') }}/index.php?option=keycloak&task=callback -JOOMLA_OIDC_END_SESSION_URL={{ OIDC.CLIENT.LOGOUT_URL }} -JOOMLA_OIDC_FALLBACK_ENABLED={{ lookup('config', application_id, 'services.sso.oidc.fallback_enabled') | string | lower }} -JOOMLA_OIDC_GROUP_ADMIN=/{{ lookup('rbac_group_path', application_id=application_id, role='administrator') }} -JOOMLA_OIDC_GROUP_EDITOR=/{{ lookup('rbac_group_path', application_id=application_id, role='editor') }} -JOOMLA_OIDC_GROUP_USER=/roles/{{ application_id }} +JOOMLA_OIDC_ISSUER_URL={{ JOOMLA_OIDC.issuer_url }} +JOOMLA_OIDC_CLIENT_ID={{ JOOMLA_OIDC.client_id }} +JOOMLA_OIDC_CLIENT_SECRET={{ JOOMLA_OIDC.client_secret | dotenv_quote }} +JOOMLA_OIDC_REDIRECT_URI={{ JOOMLA_OIDC.redirect_uri }} +JOOMLA_OIDC_END_SESSION_URL={{ JOOMLA_OIDC.end_session_url }} +JOOMLA_OIDC_FALLBACK_ENABLED={{ JOOMLA_OIDC.fallback_enabled }} +JOOMLA_OIDC_GROUP_ADMIN={{ JOOMLA_OIDC.group_admin }} +JOOMLA_OIDC_GROUP_EDITOR={{ JOOMLA_OIDC.group_editor }} +JOOMLA_OIDC_GROUP_USER={{ JOOMLA_OIDC.group_user }} {% endif %} diff --git a/roles/web-app-joomla/templates/playwright.env.j2 b/roles/web-app-joomla/templates/playwright.env.j2 index 33c5ca0605..f4a7a2c28e 100644 --- a/roles/web-app-joomla/templates/playwright.env.j2 +++ b/roles/web-app-joomla/templates/playwright.env.j2 @@ -36,3 +36,5 @@ PERSONA_BIBER_BLOCKED=true SSO_SERVICE_ENABLED={{ lookup('config', application_id, 'services.sso.enabled') | string | lower }} +{{ lookup('addon_env_flags', application_id) }} + diff --git a/roles/web-app-joomla/vars/main.yml b/roles/web-app-joomla/vars/main.yml index 9cf49406c7..af5a9a224c 100644 --- a/roles/web-app-joomla/vars/main.yml +++ b/roles/web-app-joomla/vars/main.yml @@ -2,7 +2,6 @@ # General application_id: "web-app-joomla" container_port: 80 -webserver_client_max_body_size: "{{ JOOMLA_POST_MAX_SIZE }}" entity_name: "{{ application_id | get_entity_name }}" # Joomla diff --git a/roles/web-app-keycloak/meta/services.yml b/roles/web-app-keycloak/meta/services.yml index 161451121c..41081750f0 100644 --- a/roles/web-app-keycloak/meta/services.yml +++ b/roles/web-app-keycloak/meta/services.yml @@ -6,6 +6,7 @@ logout: enabled: false shared: false ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" sync: @@ -13,16 +14,20 @@ ldap: changes: 900 full: 86400 dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: playwright-service-flag โ€” self-provider keycloak: + bond: 1 enabled: true shared: true provides: sso @@ -76,10 +81,12 @@ sso_proxy: pids_limit: 256 # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag @@ -87,5 +94,6 @@ css: recaptcha: enabled: "{{ RECAPTCHA_ENABLED | bool }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-keycloak/tasks/01_core.yml b/roles/web-app-keycloak/tasks/01_core.yml index 9502a387af..5cbe7a3ac0 100644 --- a/roles/web-app-keycloak/tasks/01_core.yml +++ b/roles/web-app-keycloak/tasks/01_core.yml @@ -13,6 +13,7 @@ name: sys-stk-full vars: compose_handlers_flush: true + webserver_extra_configuration: "{{ lookup('template', 'nginx/helper_iframes.conf.j2') }}" - name: "Load Login routines for '{{ application_id }}'" include_tasks: 04_login.yml diff --git a/roles/web-app-keycloak/vars/main.yml b/roles/web-app-keycloak/vars/main.yml index 4afa0ac8b5..14a9a9d809 100644 --- a/roles/web-app-keycloak/vars/main.yml +++ b/roles/web-app-keycloak/vars/main.yml @@ -3,7 +3,6 @@ application_id: "web-app-keycloak" entity_name: "{{ application_id | get_entity_name }}" domain: "{{ lookup('domain',application_id) }}" -webserver_extra_configuration: "{{ lookup('template', 'nginx/helper_iframes.conf.j2') }}" # Keycloak diff --git a/roles/web-app-kix/meta/services.yml b/roles/web-app-kix/meta/services.yml index addcb135a6..8830847d6d 100644 --- a/roles/web-app-kix/meta/services.yml +++ b/roles/web-app-kix/meta/services.yml @@ -1,20 +1,26 @@ --- logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -27,14 +33,17 @@ sso: - "roles/web-app-kix/user" # nocheck: playwright-service-flag โ€” verified by web-svc-css provider spec css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" # nocheck: dynamic-flag # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests redis: + bond: 1 enabled: true shared: true kix: diff --git a/roles/web-app-lam/meta/services.yml b/roles/web-app-lam/meta/services.yml index d86622e70f..eb3b10bfe7 100644 --- a/roles/web-app-lam/meta/services.yml +++ b/roles/web-app-lam/meta/services.yml @@ -1,18 +1,23 @@ --- logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" ldap: + bond: 1 enabled: true # nocheck: dynamic-flag shared: true # nocheck: dynamic-flag โ€” LDAP IS the storage backend, never local dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" # nocheck: playwright-service-flag โ€” self-provider lam: + bond: 1 enabled: true shared: true provides: lam @@ -31,6 +36,7 @@ lam: mem_limit: 256m pids_limit: 256 sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -41,8 +47,10 @@ sso: allowed_groups: - "{{ lookup('rbac_group_path', application_id='web-app-lam', role='administrator') }}" css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-listmonk/meta/services.yml b/roles/web-app-listmonk/meta/services.yml index d09b4e06c3..4a95f024cc 100644 --- a/roles/web-app-listmonk/meta/services.yml +++ b/roles/web-app-listmonk/meta/services.yml @@ -1,24 +1,31 @@ --- asset: + bond: 1 enabled: "{{ 'web-svc-asset' in group_names }}" shared: "{{ 'web-svc-asset' in group_names }}" sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" listmonk: @@ -50,15 +57,18 @@ minio: public: true # nocheck: playwright-service-flag โ€” object store, no Playwright surface seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" public: true css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag hcaptcha: enabled: "{{ HCAPTCHA_ENABLED | bool }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-littlejs/meta/services.yml b/roles/web-app-littlejs/meta/services.yml index f15e64f49a..2409e0f2b6 100644 --- a/roles/web-app-littlejs/meta/services.yml +++ b/roles/web-app-littlejs/meta/services.yml @@ -11,16 +11,20 @@ sso: enabled: false shared: false dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag javascript: + bond: 1 enabled: true shared: true littlejs: @@ -45,5 +49,6 @@ email: enabled: false shared: false prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-magento/meta/services.yml b/roles/web-app-magento/meta/services.yml index ba8f1b6af0..835bc524f3 100644 --- a/roles/web-app-magento/meta/services.yml +++ b/roles/web-app-magento/meta/services.yml @@ -1,11 +1,13 @@ --- # nocheck: seaweedfs-playwright โ€” Magento remote storage is wired into bin/magento setup:install at first boot; the role ships no templates/playwright.env.j2 (not Playwright-enabled) so the gated upload scenario cannot run here logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" # nocheck: dynamic-flag # nocheck: oidc sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oauth2 @@ -14,12 +16,15 @@ sso: host: nginx port: "8080" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" php: @@ -55,6 +60,10 @@ redis: backup: disabled: true enabled: true + cpus: "0.5" + mem_reservation: 128m + mem_limit: 512m + pids_limit: 512 # nocheck: shared # nocheck: dynamic-flag search: @@ -68,13 +77,16 @@ search: mem_limit: 2g pids_limit: 1024 seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" public: true css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" magento: diff --git a/roles/web-app-mailu/meta/services.yml b/roles/web-app-mailu/meta/services.yml index b42c0475c3..8533198fbb 100644 --- a/roles/web-app-mailu/meta/services.yml +++ b/roles/web-app-mailu/meta/services.yml @@ -1,6 +1,7 @@ --- # nocheck: seaweedfs-required โ€” object storage not implemented for this role sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc @@ -8,16 +9,20 @@ sso: email_by_username: true enable_user_creation: true logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" # nocheck: playwright-service-flag โ€” self-provider mailu: + bond: 1 enabled: true shared: true provides: email @@ -32,7 +37,7 @@ mailu: - web-app-matomo lifecycle: beta cpus: "1.0" - mem_reservation: 1g + mem_reservation: 512m mem_limit: 2g pids_limit: 1024 # nocheck: shared @@ -48,6 +53,7 @@ redis: pids_limit: 256 # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests mariadb: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-mariadb' in group_names }}" cpus: "0.8" @@ -60,7 +66,7 @@ admin: enabled: true shared: false cpus: "0.3" - mem_reservation: 512m + mem_reservation: 256m mem_limit: 1g pids_limit: 512 # nocheck: dynamic-flag @@ -69,7 +75,7 @@ imap: enabled: true shared: false cpus: "0.5" - mem_reservation: 1g + mem_reservation: 512m mem_limit: 1.5g pids_limit: 512 # nocheck: dynamic-flag @@ -78,7 +84,7 @@ smtp: enabled: true shared: false cpus: "0.5" - mem_reservation: 1g + mem_reservation: 512m mem_limit: 1.5g pids_limit: 512 # nocheck: dynamic-flag @@ -87,7 +93,7 @@ antispam: enabled: true shared: false cpus: "0.6" - mem_reservation: 1g + mem_reservation: 512m mem_limit: 1.5g pids_limit: 512 # nocheck: dynamic-flag @@ -134,7 +140,7 @@ webmail: enabled: true shared: false cpus: "0.3" - mem_reservation: 512m + mem_reservation: 256m mem_limit: 1g pids_limit: 512 # nocheck: dynamic-flag @@ -152,13 +158,15 @@ front: enabled: true shared: false cpus: "0.3" - mem_reservation: 512m + mem_reservation: 256m mem_limit: 1g pids_limit: 512 # nocheck: playwright-service-flag โ€” verified by web-svc-css provider spec css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-mailu/tasks/01_core.yml b/roles/web-app-mailu/tasks/01_core.yml index a9f4ba85cc..21f1b3f3be 100644 --- a/roles/web-app-mailu/tasks/01_core.yml +++ b/roles/web-app-mailu/tasks/01_core.yml @@ -34,6 +34,7 @@ vars: compose_handlers_flush: true webserver_vhost_flavour: "basic" + webserver_extra_configuration: "client_max_body_size 31M;" domain: "{{ MAILU_HOSTNAME }}" - name: "Flush Docker Compose handlers" diff --git a/roles/web-app-mailu/tasks/04b_create_user_token.yml b/roles/web-app-mailu/tasks/04b_create_user_token.yml index e0d5c10537..698a16e3e1 100644 --- a/roles/web-app-mailu/tasks/04b_create_user_token.yml +++ b/roles/web-app-mailu/tasks/04b_create_user_token.yml @@ -10,6 +10,9 @@ - mailu_user_existing_token is not none - mailu_user_existing_token.id is defined register: mailu_token_delete + until: mailu_token_delete.rc == 0 + retries: 30 + delay: 2 changed_when: mailu_token_delete.rc == 0 no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" @@ -44,6 +47,9 @@ args: chdir: "{{ MAILU_INSTANCE_DIR }}" register: mailu_token_creation_request + until: mailu_token_creation_request.rc == 0 + retries: 30 + delay: 2 failed_when: mailu_token_creation_request.rc != 0 changed_when: mailu_token_creation_request.rc == 0 no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" diff --git a/roles/web-app-mailu/vars/main.yml b/roles/web-app-mailu/vars/main.yml index 320e01c851..599090b460 100644 --- a/roles/web-app-mailu/vars/main.yml +++ b/roles/web-app-mailu/vars/main.yml @@ -2,7 +2,6 @@ # General application_id: "web-app-mailu" http_port: "{{ lookup('config', application_id, 'services.mailu.ports.local.http') }}" -webserver_extra_configuration: "client_max_body_size 31M;" # Database Configuration database_password: "{{ lookup('config', application_id, 'credentials.database_password') }}" # Host Directories diff --git a/roles/web-app-mastodon/meta/server.yml b/roles/web-app-mastodon/meta/server.yml index 8523efe550..b8841352c0 100644 --- a/roles/web-app-mastodon/meta/server.yml +++ b/roles/web-app-mastodon/meta/server.yml @@ -4,6 +4,9 @@ domains: - microblog.{{ DOMAIN_PRIMARY }} aliases: [] csp: + flags: + script-src-elem: + unsafe-inline: true whitelist: frame-src: - "*" diff --git a/roles/web-app-mastodon/meta/services.yml b/roles/web-app-mastodon/meta/services.yml index 5f250a8eae..a35b1ca9d0 100644 --- a/roles/web-app-mastodon/meta/services.yml +++ b/roles/web-app-mastodon/meta/services.yml @@ -1,18 +1,23 @@ --- sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: shared @@ -26,9 +31,11 @@ redis: mem_limit: 512m pids_limit: 256 postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" mastodon: + bond: 1 shared: true enabled: true image: ghcr.io/mastodon/mastodon @@ -70,12 +77,15 @@ minio: public: true # nocheck: playwright-service-flag โ€” object store, no Playwright surface seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" public: true css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-matomo/meta/services.yml b/roles/web-app-matomo/meta/services.yml index 98e429461c..d3a06ab4b5 100644 --- a/roles/web-app-matomo/meta/services.yml +++ b/roles/web-app-matomo/meta/services.yml @@ -7,23 +7,26 @@ sso: shared: false # nocheck: dynamic-flag logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" # nocheck: playwright-service-flag โ€” self-provider matomo: + bond: 1 enabled: true shared: true image: matomo - version: latest + version: "5.3.2" name: matomo min_storage: 5GB backup: no_stop_required: true cpus: 1.0 - mem_reservation: 0.5g + mem_reservation: 512m mem_limit: 1g pids_limit: 512 site_name: Matomo on {{ DOMAIN_PRIMARY }} @@ -42,12 +45,13 @@ bootstrap: - bootstrap timeout: 30 cpus: 1.0 - mem_reservation: 0.5g + mem_reservation: 256m mem_limit: 1g pids_limit: 512 min_storage: 1GB # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests mariadb: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-mariadb' in group_names }}" # nocheck: dynamic-flag @@ -58,8 +62,10 @@ redis: enabled: false # nocheck: playwright-service-flag โ€” verified by web-svc-css provider spec css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-matrix/README.md b/roles/web-app-matrix/README.md index 3a9b869f44..72db248e6f 100644 --- a/roles/web-app-matrix/README.md +++ b/roles/web-app-matrix/README.md @@ -9,7 +9,7 @@ Step into the future of communication with Matrix, a dynamic and decentralized p This role deploys a Matrix homeserver and the Element web client. Two deployment flavors are supported, selected via `services.matrix.flavor`: - `ansible` (default) wraps [matrix-docker-ansible-deploy](https://github.com/spantaleev/matrix-docker-ansible-deploy) (MDAD). MDAD runs ~70 upstream sub-roles to deliver Synapse, Element, every supported bridge, Element Call, and Jitsi. Infinito.Nexus pins MDAD by commit, renders an MDAD inventory from `services.matrix.*` knobs, and disables MDAD's bundled Traefik / Postgres / Redis so the central Infinito services back the stack. -- `compose` is the legacy in-repo Docker Compose stack. It remains installable for existing deployments but receives no new features. +- `compose` is the in-repo Docker Compose stack. It remains installable but receives no new features. The compose flavor is **deprecated**. New deployments should use the ansible flavor. @@ -33,26 +33,47 @@ Pinned MDAD upstream lives at `services.matrix.upstream.{repo,ref}`. To bump: MDAD knobs are rendered from `templates/flavor/ansible/vars.yml.j2`. Central-service consumers (postgres, redis, mailu, keycloak, openldap) are wired automatically when their `services..enabled` toggles are true. -**Runner-container encapsulation.** MDAD's `ansible-playbook setup.yml` invocation runs inside a dedicated sub-container (image: `services.matrix.runner.image`, built from `files/flavor/ansible/runner/Dockerfile`). This isolates MDAD's Ansible runtime from the Infinito-Nexus deploy container's Ansible โ€” MDAD can pin its own `ansible-core` / `community.docker` / `community.general` versions without colliding with Infinito's. The runner mounts: +**Runner-container encapsulation.** MDAD's `ansible-playbook setup.yml` invocation runs inside a dedicated sub-container (image: `services.matrix.runner.image`, built from `files/flavor/ansible/runner/Dockerfile`). This isolates MDAD's Ansible runtime from the Infinito-Nexus deploy container's Ansible, so MDAD can pin its own `ansible-core` / `community.docker` / `community.general` versions without colliding with Infinito's. The runner mounts: -- `/var/run/docker.sock` โ€” so MDAD-launched containers land in the same Docker daemon as the rest of the stack -- `/etc/systemd/system`, `/run/systemd` โ€” so MDAD's `systemctl` calls target the deploy container's systemd -- `/matrix` โ€” MDAD's data and config root -- the MDAD checkout at `{{ MATRIX_MDAD_DIR }}` โ€” read-write working copy +- `/var/run/docker.sock`, so MDAD-launched containers land in the same Docker daemon as the rest of the stack +- `/etc/systemd/system`, `/run/systemd`, so MDAD's `systemctl` calls target the deploy container's systemd +- `/matrix`, MDAD's data and config root +- the MDAD checkout at `{{ MATRIX_MDAD_DIR }}`, a read-write working copy `/usr/local/bin` is NOT bind-mounted, even though MDAD writes a few helper scripts there. The runner's `ansible-core` lives at `/opt/ansible/bin/` (out of the way) to keep the option open, but mounting the deploy container's `/usr/local/bin` over the runner causes a Python ABI mismatch segfault (host Python is built differently from `python:3.13-slim`). MDAD's host-side helper scripts are not invoked by the deploy flow. -A `mount --make-rshared /` runs once in the deploy container before the runner starts, because MDAD's systemd units bind-mount `/matrix/synapse/storage` with `bind-propagation=slave` and that requires shared/slave propagation on the source. +A `mount --make-rshared /` runs once in the deploy container before the runner starts, because MDAD's systemd units bind-mount `/matrix/synapse/storage` with `bind-propagation=slave`, which requires shared/slave propagation on the source. ### `compose` (deprecated) -Set `services.matrix.flavor: compose` to opt in. Tasks live under `tasks/flavor/compose/`, templates under `templates/flavor/compose/`. Existing volumes and credentials are reused โ€” no migration needed for in-place deploys. +Set `services.matrix.flavor: compose` to opt in. Tasks live under `tasks/flavor/compose/`, templates under `templates/flavor/compose/`. Existing volumes and credentials are reused, so in-place deploys need no migration. ## Bridge matrix Set per-bridge flags under `services.matrix.plugins.: true|false` in `meta/services.yml`. Conservative defaults: `mautrix_{signal,telegram,twitter,slack}`, `appservice_irc`, `heisenbridge`, `discord`, `gitter`, `hookshot` ON; `whatsapp`/`facebook`/`instagram`/`googlechat`/`sms` OFF. -The ansible flavor maps each true flag to the matching `matrix__enabled` MDAD var; the compose flavor uses the legacy `MATRIX_BRIDGES` loop. +The ansible flavor maps each true flag to the matching `matrix__enabled` MDAD var; the compose flavor builds the `MATRIX_BRIDGES` loop from the enabled `mechanism: bridge` addons (see Addons below). + +## Addons + +The mautrix network bridges are declared in +[`meta/addons/`](./meta/addons/) as `mechanism: bridge` addons +(requirement 026, Decision 13). Each is `required: false` and **disabled by default**; its +per-network DB password is referenced from +[`meta/schema.yml`](./meta/schema.yml) `credentials:`, never inlined. + +| Addon | Mechanism | Default state | Bridges | +|-------|-----------|---------------|---------| +| `mautrix-whatsapp` | `bridge` | disabled | external network | +| `mautrix-telegram` | `bridge` | disabled | external network | +| `mautrix-signal` | `bridge` | disabled | external network | +| `mautrix-slack` | `bridge` | disabled | external network | +| `mautrix-meta` | `bridge` | disabled | external network | + +The compose flavor derives `MATRIX_BRIDGES` from the enabled bridge addons' +`config:` blocks; the enabled/disabled split is exercised by the compose +variant in [`meta/variants.yml`](./meta/variants.yml). Coverage is via +`test-bridge-roster.js`. ## Conferencing diff --git a/roles/web-app-matrix/files/flavor/ansible/runner/bootstrap.sh b/roles/web-app-matrix/files/flavor/ansible/runner/bootstrap.sh index cd9ecd8465..300a41fc83 100644 --- a/roles/web-app-matrix/files/flavor/ansible/runner/bootstrap.sh +++ b/roles/web-app-matrix/files/flavor/ansible/runner/bootstrap.sh @@ -15,6 +15,24 @@ fi export PATH="/opt/ansible/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +PYTHON=/opt/ansible/bin/python3 +ready=0 +attempt=0 +while [ "$attempt" -lt 30 ]; do + if "$PYTHON" -c 'import _posixsubprocess, subprocess, ssl, ctypes' >/dev/null 2>&1; then + ready=1 + break + fi + attempt=$((attempt + 1)) + echo ">>> matrix-mdad-bootstrap: python runtime not ready yet (attempt ${attempt}/30), waiting" + sleep 2 +done +if [ "$ready" -ne 1 ]; then + echo "!!! matrix-mdad-bootstrap: python runtime never became importable, aborting" >&2 + "$PYTHON" -c 'import _posixsubprocess' || true + exit 1 +fi + cd /mdad START_GALAXY=$SECONDS diff --git a/roles/web-app-matrix/files/playwright/addons/mautrix-meta.spec.js b/roles/web-app-matrix/files/playwright/addons/mautrix-meta.spec.js new file mode 100644 index 0000000000..2c1e0755df --- /dev/null +++ b/roles/web-app-matrix/files/playwright/addons/mautrix-meta.spec.js @@ -0,0 +1,61 @@ +const { test, expect } = require("@playwright/test"); +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const shared = require("../_shared"); + +test.use({ ignoreHTTPSErrors: true }); + +test("mautrix-meta addon: bridge appservice registers @metabot on the partner Synapse homeserver", async ({ request }) => { + skipUnlessAddonEnabled("mautrix-meta"); + test.setTimeout(120_000); + + const { matrixBaseUrl, matrixServerName } = shared.env; + expect(matrixBaseUrl, "MATRIX_BASE_URL must be set to reach the partner homeserver").toBeTruthy(); + expect(matrixServerName, "MATRIX_SERVER_NAME must be set to address the bridge bot").toBeTruthy(); + + const botUserId = `@metabot:${matrixServerName}`; + const profileUrl = new URL( + `_matrix/client/v3/profile/${encodeURIComponent(botUserId)}`, + matrixBaseUrl.endsWith("/") ? matrixBaseUrl : `${matrixBaseUrl}/` + ).toString(); + + const matrixHost = new URL(matrixBaseUrl).host; + expect( + new URL(profileUrl).host, + "the bridge bot profile must be resolved against the partner Matrix homeserver host" + ).toBe(matrixHost); + + const response = await request.get(profileUrl, { failOnStatusCode: false }); + + expect( + response.status(), + `the partner Synapse must serve the bridge bot profile for ${botUserId}; ` + + "a 404 means the mautrix-meta appservice never registered its bot (bridge wiring broken), " + + "and a 5xx means the homeserver is unreachable โ€” either way the coupling failed and this MUST fail, not skip" + ).toBe(200); + + const body = await response.json(); + expect( + body, + "the bridge bot profile payload must come back as an object from the partner homeserver" + ).toBeTruthy(); + expect( + typeof body.displayname, + `the registered ${botUserId} must expose a displayname provisioned by the mautrix-meta appservice` + ).toBe("string"); + expect( + body.displayname, + "the Meta bridge bot displayname must be the one provisioned by the bridge registration" + ).toMatch(/meta/i); + + const controlMxid = `@definitely-not-a-meta-bot-${Date.now()}:${matrixServerName}`; + const controlUrl = new URL( + `_matrix/client/v3/profile/${encodeURIComponent(controlMxid)}`, + matrixBaseUrl.endsWith("/") ? matrixBaseUrl : `${matrixBaseUrl}/` + ).toString(); + const controlResponse = await request.get(controlUrl, { failOnStatusCode: false }); + expect( + controlResponse.status(), + `an un-bridged localpart ${controlMxid} must be unknown to Synapse (404); if it resolves, ` + + "the mautrix-meta appservice namespace is over-broad and the @metabot 200 above proves nothing" + ).toBe(404); +}); diff --git a/roles/web-app-matrix/files/playwright/addons/mautrix-signal.spec.js b/roles/web-app-matrix/files/playwright/addons/mautrix-signal.spec.js new file mode 100644 index 0000000000..c78d84c11d --- /dev/null +++ b/roles/web-app-matrix/files/playwright/addons/mautrix-signal.spec.js @@ -0,0 +1,41 @@ +const { test, expect } = require("@playwright/test"); +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const shared = require("../_shared"); + +test.use({ ignoreHTTPSErrors: true }); + +test("mautrix-signal addon: bridge appservice bot is provisioned and reachable on the Synapse homeserver", async ({ request }) => { + skipUnlessAddonEnabled("mautrix-signal"); + test.setTimeout(60_000); + + const { matrixBaseUrl, matrixServerName } = shared.env; + expect(matrixBaseUrl, "MATRIX_BASE_URL must be set").toBeTruthy(); + expect(matrixServerName, "MATRIX_SERVER_NAME must be set").toBeTruthy(); + + const botUserId = `@signalbot:${matrixServerName}`; + const profileUrl = (userId) => + `${matrixBaseUrl}/_matrix/client/v3/profile/${encodeURIComponent(userId)}`; + + const botResponse = await request.get(profileUrl(botUserId), { failOnStatusCode: false }); + expect( + botResponse.status(), + `the mautrix-signal appservice bot ${botUserId} must be registered on the Synapse homeserver โ€” a 404/M_NOT_FOUND means the bridge registration never landed` + ).toBe(200); + + const botProfile = await botResponse.json(); + expect( + botProfile, + `${botUserId} profile must expose the provisioned bridge bot display name, proving the appservice (not just a config value) reached Synapse` + ).toHaveProperty("displayname"); + expect( + String(botProfile.displayname || ""), + `${botUserId} display name must be the Signal bridge bot, distinct from an empty/local placeholder` + ).toMatch(/signal/i); + + const absentBotUserId = `@signalbot-not-provisioned-${Date.now()}:${matrixServerName}`; + const absentResponse = await request.get(profileUrl(absentBotUserId), { failOnStatusCode: false }); + expect( + absentResponse.status(), + "control: an unregistered bridge bot must return 404 on the same homeserver โ€” otherwise the 200 above is meaningless" + ).toBe(404); +}); diff --git a/roles/web-app-matrix/files/playwright/addons/mautrix-slack.spec.js b/roles/web-app-matrix/files/playwright/addons/mautrix-slack.spec.js new file mode 100644 index 0000000000..9f24fe24f2 --- /dev/null +++ b/roles/web-app-matrix/files/playwright/addons/mautrix-slack.spec.js @@ -0,0 +1,49 @@ +const { test, expect } = require("@playwright/test"); +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const shared = require("../_shared"); + +test.use({ ignoreHTTPSErrors: true }); + +test("mautrix-slack addon: Slack bridge appservice bot is provisioned and reachable on Synapse", async ({ request }) => { + skipUnlessAddonEnabled("mautrix-slack"); + test.setTimeout(60_000); + + const matrixBaseUrl = shared.env.matrixBaseUrl; + const matrixServerName = shared.env.matrixServerName; + expect(matrixBaseUrl, "MATRIX_BASE_URL must be set for the bridge probe").toBeTruthy(); + expect(matrixServerName, "MATRIX_SERVER_NAME must be set for the bridge probe").toBeTruthy(); + + const botUserId = `@slackbot:${matrixServerName}`; + const profileUrl = `${matrixBaseUrl.replace(/\/$/, "")}/_matrix/client/v3/profile/${encodeURIComponent(botUserId)}`; + + const response = await request.get(profileUrl, { failOnStatusCode: false }); + const status = response.status(); + const body = await response.text(); + + expect( + status, + `the mautrix-slack appservice bot ${botUserId} must be reachable on Synapse (the bridge actually registered its appservice with the homeserver). HTTP ${status}: ${body.slice(0, 300)}` + ).toBeLessThan(500); + + expect( + status, + `Synapse must know the mautrix-slack bridge bot ${botUserId}. A 404/M_NOT_FOUND means the bridge's appservice registration never landed โ€” the coupling failed to provision, so this MUST fail (not skip). HTTP ${status}: ${body.slice(0, 300)}` + ).not.toBe(404); + + expect( + status, + `the mautrix-slack bridge bot profile lookup must succeed. HTTP ${status}: ${body.slice(0, 300)}` + ).toBe(200); + + let profile; + try { + profile = JSON.parse(body); + } catch (e) { + throw new Error(`mautrix-slack bridge bot profile must be valid JSON. HTTP ${status}: ${body.slice(0, 300)}`, { cause: e }); + } + + expect( + profile.displayname, + "the provisioned mautrix-slack bridge bot must carry its Slack-bridge displayname (proves the bridge โ€” not a stray Matrix account โ€” owns @slackbot)" + ).toMatch(/slack/i); +}); diff --git a/roles/web-app-matrix/files/playwright/addons/mautrix-telegram.spec.js b/roles/web-app-matrix/files/playwright/addons/mautrix-telegram.spec.js new file mode 100644 index 0000000000..36cdb8f448 --- /dev/null +++ b/roles/web-app-matrix/files/playwright/addons/mautrix-telegram.spec.js @@ -0,0 +1,36 @@ +const { test, expect } = require("@playwright/test"); +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const shared = require("../_shared"); + +test.use({ ignoreHTTPSErrors: true }); + +const TELEGRAM_BOT_LOCALPART = "telegrambot"; + +test("mautrix-telegram addon: appservice bot is provisioned on the Synapse homeserver", async ({ request }) => { + skipUnlessAddonEnabled("mautrix-telegram"); + test.setTimeout(60_000); + + const matrixBaseUrl = shared.env.matrixBaseUrl; + const matrixServerName = shared.env.matrixServerName; + expect(matrixBaseUrl, "MATRIX_BASE_URL must be set").toBeTruthy(); + expect(matrixServerName, "MATRIX_SERVER_NAME must be set").toBeTruthy(); + + const botUserId = `@${TELEGRAM_BOT_LOCALPART}:${matrixServerName}`; + const profileUrl = + `${matrixBaseUrl}/_matrix/client/v3/profile/${encodeURIComponent(botUserId)}`; + + const response = await request.get(profileUrl, { failOnStatusCode: false }); + + expect( + response.status(), + `the mautrix-telegram bridge appservice bot ${botUserId} must be registered on Synapse (${matrixBaseUrl}). ` + + `A 404 means the appservice registration never landed in the homeserver โ€” the bridge coupling failed to provision. ` + + `A 5xx means the homeserver is unhealthy. Got HTTP ${response.status()}.` + ).toBe(200); + + const profile = await response.json().catch(() => null); + expect( + profile && typeof profile === "object" && !Array.isArray(profile), + `Synapse must return the telegram bridge bot's profile JSON object for ${botUserId}, proving the appservice bot user exists` + ).toBe(true); +}); diff --git a/roles/web-app-matrix/files/playwright/addons/mautrix-whatsapp.spec.js b/roles/web-app-matrix/files/playwright/addons/mautrix-whatsapp.spec.js new file mode 100644 index 0000000000..1c435a07d7 --- /dev/null +++ b/roles/web-app-matrix/files/playwright/addons/mautrix-whatsapp.spec.js @@ -0,0 +1,49 @@ +const { test, expect } = require("@playwright/test"); +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const shared = require("../_shared"); + +test.use({ ignoreHTTPSErrors: true }); + +test("mautrix-whatsapp addon: bridge appservice registers @whatsappbot on the partner Synapse homeserver", async ({ request }) => { + skipUnlessAddonEnabled("mautrix-whatsapp"); + test.setTimeout(120_000); + + const { matrixBaseUrl, matrixServerName } = shared.env; + expect(matrixBaseUrl, "MATRIX_BASE_URL must be set to reach the partner homeserver").toBeTruthy(); + expect(matrixServerName, "MATRIX_SERVER_NAME must be set to address the bridge bot").toBeTruthy(); + + const botUserId = `@whatsappbot:${matrixServerName}`; + const profileUrl = new URL( + `_matrix/client/v3/profile/${encodeURIComponent(botUserId)}`, + matrixBaseUrl.endsWith("/") ? matrixBaseUrl : `${matrixBaseUrl}/` + ).toString(); + + const matrixHost = new URL(matrixBaseUrl).host; + expect( + new URL(profileUrl).host, + "the bridge bot profile must be resolved against the partner Matrix homeserver host" + ).toBe(matrixHost); + + const response = await request.get(profileUrl, { failOnStatusCode: false }); + + expect( + response.status(), + `the partner Synapse must serve the bridge bot profile for ${botUserId}; ` + + "a 404 means the mautrix-whatsapp appservice never registered its bot (bridge wiring broken), " + + "and a 5xx means the homeserver is unreachable โ€” either way the coupling failed and this MUST fail, not skip" + ).toBe(200); + + const body = await response.json(); + expect( + body, + "the bridge bot profile payload must come back as an object from the partner homeserver" + ).toBeTruthy(); + expect( + typeof body.displayname, + `the registered ${botUserId} must expose a displayname provisioned by the mautrix-whatsapp appservice` + ).toBe("string"); + expect( + body.displayname, + "the WhatsApp bridge bot displayname must be the one provisioned by the bridge registration" + ).toMatch(/whatsapp/i); +}); diff --git a/roles/web-app-matrix/filter_plugins/bridge_filters.py b/roles/web-app-matrix/filter_plugins/bridge_filters.py deleted file mode 100644 index 6d0470f8df..0000000000 --- a/roles/web-app-matrix/filter_plugins/bridge_filters.py +++ /dev/null @@ -1,14 +0,0 @@ -def filter_enabled_bridges(bridges, plugins): - """ - Return only those bridge definitions whose 'bridge_name' is set to True in plugins. - :param bridges: list of dicts, each with a 'bridge_name' key - :param plugins: dict mapping bridge_name to a boolean - """ - return [b for b in bridges if plugins.get(b["bridge_name"], False)] - - -class FilterModule: - def filters(self): - return { - "filter_enabled_bridges": filter_enabled_bridges, - } diff --git a/roles/web-app-matrix/meta/addons/mautrix-meta.yml b/roles/web-app-matrix/meta/addons/mautrix-meta.yml new file mode 100644 index 0000000000..1cb6eea3b4 --- /dev/null +++ b/roles/web-app-matrix/meta/addons/mautrix-meta.yml @@ -0,0 +1,14 @@ +--- +enabled: false +required: false +mechanism: bridge +source: upstream +update: + monitored: true + catalog: mautrix + upstream_id: mautrix-meta +config: + bridge_name: "meta" + database_username: "mautrix_meta_bridge" + database_name: "mautrix_meta_bridge" + database_password: "{{ lookup('config', 'web-app-matrix', 'credentials.mautrix_meta_bridge_database_password') }}" diff --git a/roles/web-app-matrix/meta/addons/mautrix-signal.yml b/roles/web-app-matrix/meta/addons/mautrix-signal.yml new file mode 100644 index 0000000000..14ce98a06e --- /dev/null +++ b/roles/web-app-matrix/meta/addons/mautrix-signal.yml @@ -0,0 +1,14 @@ +--- +enabled: false +required: false +mechanism: bridge +source: upstream +update: + monitored: true + catalog: mautrix + upstream_id: mautrix-signal +config: + bridge_name: "signal" + database_username: "mautrix_signal_bridge" + database_name: "mautrix_signal_bridge" + database_password: "{{ lookup('config', 'web-app-matrix', 'credentials.mautrix_signal_bridge_database_password') }}" diff --git a/roles/web-app-matrix/meta/addons/mautrix-slack.yml b/roles/web-app-matrix/meta/addons/mautrix-slack.yml new file mode 100644 index 0000000000..e6bb7c8e4c --- /dev/null +++ b/roles/web-app-matrix/meta/addons/mautrix-slack.yml @@ -0,0 +1,14 @@ +--- +enabled: false +required: false +mechanism: bridge +source: upstream +update: + monitored: true + catalog: mautrix + upstream_id: mautrix-slack +config: + bridge_name: "slack" + database_username: "mautrix_slack_bridge" + database_name: "mautrix_slack_bridge" + database_password: "{{ lookup('config', 'web-app-matrix', 'credentials.mautrix_slack_bridge_database_password') }}" diff --git a/roles/web-app-matrix/meta/addons/mautrix-telegram.yml b/roles/web-app-matrix/meta/addons/mautrix-telegram.yml new file mode 100644 index 0000000000..7e2fb85be9 --- /dev/null +++ b/roles/web-app-matrix/meta/addons/mautrix-telegram.yml @@ -0,0 +1,14 @@ +--- +enabled: false +required: false +mechanism: bridge +source: upstream +update: + monitored: true + catalog: mautrix + upstream_id: mautrix-telegram +config: + bridge_name: "telegram" + database_username: "mautrix_telegram_bridge" + database_name: "mautrix_telegram_bridge" + database_password: "{{ lookup('config', 'web-app-matrix', 'credentials.mautrix_telegram_bridge_database_password') }}" diff --git a/roles/web-app-matrix/meta/addons/mautrix-whatsapp.yml b/roles/web-app-matrix/meta/addons/mautrix-whatsapp.yml new file mode 100644 index 0000000000..e8313353ad --- /dev/null +++ b/roles/web-app-matrix/meta/addons/mautrix-whatsapp.yml @@ -0,0 +1,14 @@ +--- +enabled: false +required: false +mechanism: bridge +source: upstream +update: + monitored: true + catalog: mautrix + upstream_id: mautrix-whatsapp +config: + bridge_name: "whatsapp" + database_username: "mautrix_whatsapp_bridge" + database_name: "mautrix_whatsapp_bridge" + database_password: "{{ lookup('config', 'web-app-matrix', 'credentials.mautrix_whatsapp_bridge_database_password') }}" diff --git a/roles/web-app-matrix/meta/schema.yml b/roles/web-app-matrix/meta/schema.yml index be5a65e96b..8a3d7eb59f 100644 --- a/roles/web-app-matrix/meta/schema.yml +++ b/roles/web-app-matrix/meta/schema.yml @@ -36,12 +36,8 @@ credentials: description: Matrix user password used by the ChatGPT bridge algorithm: sha256 validation: ^[a-f0-9]{64}$ - mautrix_facebook_bridge_database_password: - description: Database password for the mautrix-facebook bridge - algorithm: bcrypt - validation: ^\$2[aby]\$.{56}$ - mautrix_instagram_bridge_database_password: - description: Database password for the mautrix-instagram bridge + mautrix_meta_bridge_database_password: + description: Database password for the mautrix-meta bridge (Facebook Messenger + Instagram) algorithm: bcrypt validation: ^\$2[aby]\$.{56}$ mautrix_signal_bridge_database_password: diff --git a/roles/web-app-matrix/meta/services.yml b/roles/web-app-matrix/meta/services.yml index fd74609ce1..b8920d45d0 100644 --- a/roles/web-app-matrix/meta/services.yml +++ b/roles/web-app-matrix/meta/services.yml @@ -1,29 +1,37 @@ --- sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: playwright-service-flag โ€” TURN/STUN server, no Playwright surface (provider: web-svc-coturn) coturn: + bond: 1 enabled: "{{ 'web-svc-coturn' in group_names }}" shared: "{{ 'web-svc-coturn' in group_names }}" # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests postgres: + bond: 1 enabled: true # nocheck: dynamic-flag โ€” matrix always needs a database shared: "{{ 'svc-db-postgres' in group_names }}" synapse: @@ -66,14 +74,17 @@ matrix-chatgpt-bot: pids_limit: 256 # nocheck: playwright-service-flag โ€” object store, no Playwright surface (provider: web-app-seaweedfs) seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" public: true # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" native_metrics: @@ -82,7 +93,11 @@ prometheus: service_key: synapse communication: channel: true +# nocheck: playwright-service-flag: shared provider entity; the app itself is covered by this role's own Playwright spec matrix: + bond: 1 + enabled: true + shared: true flavor: ansible upstream: repo: https://github.com/spantaleev/matrix-docker-ansible-deploy diff --git a/roles/web-app-matrix/meta/variants.yml b/roles/web-app-matrix/meta/variants.yml index 6b565f76fd..8c978e0927 100644 --- a/roles/web-app-matrix/meta/variants.yml +++ b/roles/web-app-matrix/meta/variants.yml @@ -71,7 +71,18 @@ enabled: false shared: false -- services: +- addons: + mautrix-whatsapp: + enabled: true + mautrix-telegram: + enabled: true + mautrix-signal: + enabled: true + mautrix-slack: + enabled: true + mautrix-meta: + enabled: true + services: matrix: flavor: compose seaweedfs: diff --git a/roles/web-app-matrix/tasks/01_core.yml b/roles/web-app-matrix/tasks/01_core.yml new file mode 100644 index 0000000000..69b92fa830 --- /dev/null +++ b/roles/web-app-matrix/tasks/01_core.yml @@ -0,0 +1,21 @@ +--- +- include_tasks: utils/once/flag.yml + +- name: "set matrix flavor" + ansible.builtin.set_fact: + matrix_flavor: "{{ lookup('config', application_id, 'services.matrix.flavor', 'ansible') }}" + changed_when: false + +- name: "assert matrix flavor is one of: ansible | compose" + ansible.builtin.assert: + that: + - matrix_flavor in ['ansible', 'compose'] + fail_msg: "services.matrix.flavor must be 'ansible' or 'compose' (got '{{ matrix_flavor }}')" + +- name: "dispatch to ansible flavor (matrix-docker-ansible-deploy upstream)" + include_tasks: flavor/ansible/main.yml + when: matrix_flavor == 'ansible' + +- name: "dispatch to compose flavor (deprecated; in-repo compose stack)" + include_tasks: flavor/compose/main.yml + when: matrix_flavor == 'compose' diff --git a/roles/web-app-matrix/tasks/flavor/compose/01_docker.yml b/roles/web-app-matrix/tasks/flavor/compose/01_docker.yml index d752f17b9f..ca589f47cf 100644 --- a/roles/web-app-matrix/tasks/flavor/compose/01_docker.yml +++ b/roles/web-app-matrix/tasks/flavor/compose/01_docker.yml @@ -73,6 +73,13 @@ async: "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" +- name: "Restart synapse so it loads the freshly generated bridge appservice registrations" + command: + cmd: "{{ BIN_COMPOSE }} restart synapse" + chdir: "{{ lookup('container', application_id, 'directories.instance') }}" + changed_when: true + when: MATRIX_BRIDGES | length > 0 + - name: wait for Synapse client/versions endpoint uri_retry: url: "http://127.0.0.1:{{ lookup('config', application_id, 'services.synapse.ports.local.http') }}/_matrix/client/versions" diff --git a/roles/web-app-matrix/tasks/flavor/compose/02_create-and-seed-database.yml b/roles/web-app-matrix/tasks/flavor/compose/02_create-and-seed-database.yml index 114affa384..4f486003c3 100644 --- a/roles/web-app-matrix/tasks/flavor/compose/02_create-and-seed-database.yml +++ b/roles/web-app-matrix/tasks/flavor/compose/02_create-and-seed-database.yml @@ -4,21 +4,29 @@ # - database_username # - database_password +- name: "Freeze database consumer id + provider role before provider include" + ansible.builtin.set_fact: + _matrix_db_consumer_id: "{{ application_id }}" + _matrix_db_provider_role: "{{ lookup('database', application_id, 'id') }}" + changed_when: false + - name: "Create DB/user for '{{ database_name }}' in central DB" include_role: - name: "{{ lookup('database', application_id, 'id') }}" + name: "{{ _matrix_db_provider_role }}" + public: false vars: database_init: true - database_consumer_id: "{{ application_id }}" - database_type: "{{ lookup('database', application_id, 'type') }}" - when: lookup('database', application_id, 'shared') | bool + compose_repository_pull: false + database_consumer_id: "{{ _matrix_db_consumer_id }}" + database_type: "{{ lookup('database', _matrix_db_consumer_id, 'type') }}" + when: lookup('database', _matrix_db_consumer_id, 'shared') | bool - name: "Seed database entry to backup routine" include_tasks: "{{ playbook_dir }}/roles/svc-bkp-container-2-local/tasks/04_seed-database-to-backup.yml" vars: - database_consumer_id: "{{ application_id }}" - database_type: "{{ lookup('database', application_id, 'type') }}" - database_instance: "{{ lookup('database', application_id, 'instance') }}" + database_consumer_id: "{{ _matrix_db_consumer_id }}" + database_type: "{{ lookup('database', _matrix_db_consumer_id, 'type') }}" + database_instance: "{{ lookup('database', _matrix_db_consumer_id, 'instance') }}" database_host: "{{ MATRIX_POSTGRES_HOST }}" database_port: "{{ MATRIX_POSTGRES_PORT }}" database_name: "{{ database_name }}" diff --git a/roles/web-app-matrix/tasks/flavor/compose/main.yml b/roles/web-app-matrix/tasks/flavor/compose/main.yml index fbbbdabd52..b4ca344ea3 100644 --- a/roles/web-app-matrix/tasks/flavor/compose/main.yml +++ b/roles/web-app-matrix/tasks/flavor/compose/main.yml @@ -1,11 +1,21 @@ --- -- name: Load bridges configuration - include_vars: - file: "bridges.yml" - -- name: Filter enabled bridges and register as fact +- name: Build enabled bridge list from meta/addons/ set_fact: - MATRIX_BRIDGES: "{{ bridges_configuration | filter_enabled_bridges(lookup('config', application_id, 'services.matrix.plugins')) }}" + MATRIX_BRIDGES: >- + {{ lookup('config', application_id, 'addons') + | dict2items + | selectattr('value.mechanism', 'equalto', 'bridge') + | selectattr('value.enabled', 'equalto', true) + | map(attribute='value.config') + | list }} + changed_when: false + +- name: "Drop the telegram bridge when its Telegram API credentials are absent (api_hash is required at startup)" + set_fact: + MATRIX_BRIDGES: "{{ MATRIX_BRIDGES | rejectattr('bridge_name', 'equalto', 'telegram') | list }}" + when: >- + (lookup('config', application_id, 'credentials.mautrix_telgegram_bridge_api_id', '') | default('', true) | string | trim | length == 0) + or (lookup('config', application_id, 'credentials.mautrix_telgegram_bridge_api_pin', '') | default('', true) | string | trim | length == 0) changed_when: false - name: "compose-flavor: docker stack for '{{ application_id }}'" diff --git a/roles/web-app-matrix/tasks/main.yml b/roles/web-app-matrix/tasks/main.yml index a1ee834aff..224fe4af29 100644 --- a/roles/web-app-matrix/tasks/main.yml +++ b/roles/web-app-matrix/tasks/main.yml @@ -1,19 +1,6 @@ --- -- name: "set matrix flavor" - ansible.builtin.set_fact: - matrix_flavor: "{{ lookup('config', application_id, 'services.matrix.flavor', 'ansible') }}" - changed_when: false - -- name: "assert matrix flavor is one of: ansible | compose" - ansible.builtin.assert: - that: - - matrix_flavor in ['ansible', 'compose'] - fail_msg: "services.matrix.flavor must be 'ansible' or 'compose' (got '{{ matrix_flavor }}')" - -- name: "dispatch to ansible flavor (matrix-docker-ansible-deploy upstream)" - include_tasks: flavor/ansible/main.yml - when: matrix_flavor == 'ansible' - -- name: "dispatch to compose flavor (deprecated; in-repo compose stack)" - include_tasks: flavor/compose/main.yml - when: matrix_flavor == 'compose' +- name: "Execute Role (once)" + include_tasks: 01_core.yml + when: run_once_web_app_matrix is not defined + vars: + application_id: "web-app-matrix" diff --git a/roles/web-app-matrix/templates/flavor/compose/mautrix/facebook.config.yml.j2 b/roles/web-app-matrix/templates/flavor/compose/mautrix/facebook.config.yml.j2 deleted file mode 100644 index e170e1577a..0000000000 --- a/roles/web-app-matrix/templates/flavor/compose/mautrix/facebook.config.yml.j2 +++ /dev/null @@ -1,449 +0,0 @@ -# Homeserver details -homeserver: - # The address that this appservice can use to connect to the homeserver. - address: http://synapse:{{ MATRIX_SYNAPSE_CONTAINER_PORT }} - # The domain of the homeserver (for MXIDs, etc). - domain: {{ MATRIX_SERVER_NAME }} - # Whether or not to verify the SSL certificate of the homeserver. - # Only applies if address starts with https:// - verify_ssl: true - # What software is the homeserver running? - # Standard Matrix homeservers like Synapse, Dendrite and Conduit should just use "standard" here. - software: standard - # Number of retries for all HTTP requests if the homeserver isn't reachable. - http_retry_count: 4 - # The URL to push real-time bridge status to. - # If set, the bridge will make POST requests to this URL whenever a user's Facebook MQTT connection state changes. - # The bridge will use the appservice as_token to authorize requests. - status_endpoint: null - # Endpoint for reporting per-message status. - message_send_checkpoint_endpoint: null - # Whether asynchronous uploads via MSC2246 should be enabled for media. - # Requires a media repo that supports MSC2246. - async_media: false - -# Application service host/registration related details -# Changing these values requires regeneration of the registration. -appservice: - # The address that the homeserver can use to connect to this appservice. - address: http://mautrix-facebook:29319 - - # The hostname and port where this appservice should listen. - hostname: 0.0.0.0 - port: 29319 - # The maximum body size of appservice API requests (from the homeserver) in mebibytes - # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s - max_body_size: 1 - - # The full URI to the database. SQLite and Postgres are supported. - # Format examples: - # SQLite: sqlite:filename.db - # Postgres: postgres://username:password@hostname/dbname - database: postgres://mautrix_facebook_bridge:{{ lookup('config', application_id, 'credentials.mautrix_facebook_bridge_database_password')}}@{{ MATRIX_POSTGRES_HOST }}/mautrix_facebook_bridge - # Additional arguments for asyncpg.create_pool() or sqlite3.connect() - # https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool - # https://docs.python.org/3/library/sqlite3.html#sqlite3.connect - # For sqlite, min_size is used as the connection thread pool size and max_size is ignored. - # Additionally, SQLite supports init_commands as an array of SQL queries to run on connect (e.g. to set PRAGMAs). - database_opts: - min_size: 1 - max_size: 10 - - # Public part of web server for out-of-Matrix interaction with the bridge. - public: - # Whether or not the public-facing endpoints should be enabled. - enabled: false - # The prefix to use in the public-facing endpoints. - prefix: /public - # The base URL where the public-facing endpoints are available. The prefix is not added - # implicitly. - external: https://example.com/public - # Shared secret for integration managers such as mautrix-manager. - # If set to "generate", a random string will be generated on the next startup. - # If null, integration manager access to the API will not be possible. - shared_secret: generate - # Allow logging in within Matrix. If false, users can only log in using the web interface. - allow_matrix_login: true - - # The unique ID of this appservice. - id: facebook - # Username of the appservice bot. - bot_username: facebookbot - # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty - # to leave display name/avatar as-is. - bot_displayname: Facebook bridge bot - bot_avatar: mxc://maunium.net/ygtkteZsXnGJLJHRchUwYWak - - # Whether or not to receive ephemeral events via appservice transactions. - # Requires MSC2409 support (i.e. Synapse 1.22+). - # You should disable bridge -> sync_with_custom_puppets when this is enabled. - ephemeral_events: true - - # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify. - as_token: "This value is generated when generating the registration" - hs_token: "This value is generated when generating the registration" - -# Segment-compatible analytics endpoint for tracking some events, like provisioning API login and encryption errors. -analytics: - # Hostname of the tracking server. The path is hardcoded to /v1/track - host: api.segment.io - # API key to send with tracking requests. Tracking is disabled if this is null. - token: null - # Optional user ID for tracking events. If null, defaults to using Matrix user ID. - user_id: null - -# Prometheus telemetry config. Requires prometheus-client to be installed. -metrics: - enabled: false - listen_port: 8000 - -# Manhole config. -manhole: - # Whether or not opening the manhole is allowed. - enabled: false - # The path for the unix socket. - path: /var/tmp/mautrix-facebook.manhole - # The list of UIDs who can be added to the whitelist. - # If empty, any UIDs can be specified in the open-manhole command. - whitelist: - - 0 - -# Bridge config -bridge: - # Localpart template of MXIDs for Facebook users. - # {userid} is replaced with the user ID of the Facebook user. - username_template: "facebook_{userid}" - # Displayname template for Facebook users. - # {displayname} is replaced with the display name of the Facebook user - # as defined below in displayname_preference. - # Keys available for displayname_preference are also available here. - displayname_template: "{displayname} (FB)" - # Available keys: - # "name" (full name) - # "first_name" - # "last_name" - # "nickname" - # "own_nickname" (user-specific!) - displayname_preference: - - name - - first_name - - # The prefix for commands. Only required in non-management rooms. - command_prefix: "!fb" - - # Whether or not the Facebook users of logged in Matrix users should be - # invited to private chats when the user sends a message from another client. - invite_own_puppet_to_pm: false - # Whether or not to use /sync to get presence, read receipts and typing notifications - # when double puppeting is enabled - sync_with_custom_puppets: false - # Whether or not to update the m.direct account data event when double puppeting is enabled. - # Note that updating the m.direct event is not atomic (except with mautrix-asmux) - # and is therefore prone to race conditions. - sync_direct_chat_list: false - # Servers to always allow double puppeting from - double_puppet_server_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_SYNAPSE_DOMAIN }} - # Allow using double puppeting from any server with a valid client .well-known file. - double_puppet_allow_discovery: false - # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth - # - # If set, custom puppets will be enabled automatically for local users - # instead of users having to find an access token and run `login-matrix` - # manually. - # If using this for other servers than the bridge's server, - # you must also set the URL in the double_puppet_server_map. - login_shared_secret_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_REGISTRATION_SHARED_SECRET }} - # Should presence from Facebook be bridged? This doesn't use the same API as the Android app, - # so it might be more suspicious to Facebook. - presence_from_facebook: false - # Whether or not to update avatars when syncing all contacts at startup. - update_avatar_initial_sync: true - - # Whether or not the bridge should send a read receipt from the bridge bot when a message has - # been sent to Facebook. - delivery_receipts: false - # Whether or not delivery errors should be reported as messages in the Matrix room. - delivery_error_reports: true - # Whether the bridge should send the message status as a custom com.beeper.message_send_status event. - message_status_events: false - # Whether to allow inviting arbitrary mxids to portal rooms - allow_invites: false - # Whether or not created rooms should have federation enabled. - # If false, created portal rooms will never be federated. - federate_rooms: true - # Settings for backfilling messages from Facebook. - backfill: - # Allow backfilling at all? - enable: true - # Use MSC2716 for backfilling? If this is disabled, backfilling only happens when syncing threads, - # and the incremental settings below don't apply. - # - # This requires a server with MSC2716 support, which is currently an experimental feature in Synapse. - # It can be enabled by setting experimental_features -> msc2716_enabled to true in homeserver.yaml. - msc2716: false - # Use double puppets for backfilling? - # - # If using MSC2716, the double puppets must be in the appservice's user ID namespace - # (because the bridge can't use the double puppet access token with batch sending). - # - # Even without MSC2716, bridging old messages with correct timestamps requires the double - # puppets to be in an appservice namespace, or the server to be modified to allow - # overriding timestamps anyway. - double_puppet_backfill: false - # The maximum number of conversations that should be synced. - # Other conversations will be backfilled on demand when the start PM - # provisioning endpoint is used or when a message comes in from that - # chat. - # If set to -1, all conversations will by synced. - max_conversations: 20 - # The minimum amount of time to wait between syncing each thread. This - # helps avoid situations where you sync too quickly. - min_sync_thread_delay: 5 - # If this value is greater than 0, then if the conversation's last - # message was more than this number of hours ago, then the conversation - # will automatically be marked it as read. - # Conversations that have a last message that is less than this number - # of hours ago will have their unread status synced from Facebook. - unread_hours_threshold: 0 - - # Settings for how quickly to backoff when rate-limits are encountered - # while backfilling. - backoff: - # How many seconds to wait after getting rate limited during a - # thread list fetch. - thread_list: 300 - # How many seconds to wait after getting rate limited during a - # message history fetch. - message_history: 300 - - # Settings for backfills. - # - # During initial/incremental sync, the entirety of the thread that is - # available will be backfilled. For example, on initial sync, about 20 - # messages are included for each thread in the thread list returned by - # the server. After that, incremental backfills will be run for each of - # the portals in a round-robin fashion until all portals have been - # backfilled as configured below. - incremental: - # The maximum number of pages to backfill per batch. - max_pages: 10 - # The maximum number of total pages to backfill per portal. - # If set to -1, infinite pages will be synced. - max_total_pages: -1 - # The number of seconds to wait between backfilling each page. - page_delay: 5 - # The number of seconds to wait after backfilling the batch of - # messages. - post_batch_delay: 20 - - periodic_reconnect: - # Interval in seconds in which to automatically reconnect all users. - # This can be used to automatically mitigate the bug where Facebook stops sending messages. - # Set to -1 to disable periodic reconnections entirely. - # Set to a list of two items to randomize the interval (min, max). - interval: -1 - # What to do in periodic reconnects. Either "refresh" or "reconnect" - mode: refresh - # Should even disconnected users be reconnected? - always: false - # Only reconnect if the user has been connected for longer than this value - min_connected_time: 0 - # The number of seconds that a disconnection can last without triggering an automatic re-sync - # and missed message backfilling when reconnecting. - # Set to 0 to always re-sync, or -1 to never re-sync automatically. - resync_max_disconnected_time: 5 - # The maximum number of conversations that should be synced when we get a - # message sync error. In general, 1 page (20) is sufficient. - max_startup_thread_sync_count: 20 - # Whether or not temporary disconnections should send notices to the notice room. - # If this is false, disconnections will never send messages and connections will only send - # messages if it was disconnected for more than resync_max_disconnected_time seconds. - temporary_disconnect_notices: false - # Disable bridge notices entirely - disable_bridge_notices: false - # Should Matrix m.notice-type messages be bridged to Facebook? - bridge_matrix_notices: true - on_reconnection_fail: - # What to do if a reconnection attempt fails? Options: reconnect, refresh, null - action: reconnect - # Seconds to wait before attempting to refresh the connection, set a list of two items to - # to randomize the interval (min, max). - wait_for: 0 - # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run. - # This field will automatically be changed back to false after it, - # except if the config file is not writable. - resend_bridge_info: false - # When using double puppeting, should muted chats be muted in Matrix? - mute_bridging: false - # Whether or not mute status and tags should only be bridged when the portal room is created. - tag_only_on_create: true - # If set to true, downloading media from the CDN will use a plain aiohttp client without the usual headers or - # other configuration. This may be useful if you don't want to use the default proxy for large files. - sandbox_media_download: false - # URL to call to retrieve a proxy URL from (defaults to the http_proxy environment variable). - get_proxy_api_url: null - # Whether to explicitly set the avatar and room name for private chat portal rooms. - # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms. - # If set to `always`, all DM rooms will have explicit names and avatars set. - # If set to `never`, DM rooms will never have names and avatars set. - private_chat_portal_meta: default - # Disable generating reply fallbacks? Some extremely bad clients still rely on them, - # but they're being phased out and will be completely removed in the future. - disable_reply_fallbacks: false - - # End-to-bridge encryption support options. - # - # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info. - encryption: - # Allow encryption, work in group chat rooms with e2ee enabled - allow: false - # Default to encryption, force-enable encryption in all portals the bridge creates - # This will cause the bridge bot to be in private chats for the encryption to work properly. - default: false - # Whether to use MSC2409/MSC3202 instead of /sync long polling for receiving encryption-related data. - appservice: false - # Require encryption, drop any unencrypted messages. - require: false - # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled. - # You must use a client that supports requesting keys from other users to use this feature. - allow_key_sharing: false - # Options for deleting megolm sessions from the bridge. - delete_keys: - # Beeper-specific: delete outbound sessions when hungryserv confirms - # that the user has uploaded the key to key backup. - delete_outbound_on_ack: false - # Don't store outbound sessions in the inbound table. - dont_store_outbound: false - # Ratchet megolm sessions forward after decrypting messages. - ratchet_on_decrypt: false - # Delete fully used keys (index >= max_messages) after decrypting messages. - delete_fully_used_on_decrypt: false - # Delete previous megolm sessions from same device when receiving a new one. - delete_prev_on_new_session: false - # Delete megolm sessions received from a device when the device is deleted. - delete_on_device_delete: false - # Periodically delete megolm sessions when 2x max_age has passed since receiving the session. - periodically_delete_expired: false - # Delete inbound megolm sessions that don't have the received_at field used for - # automatic ratcheting and expired session deletion. This is meant as a migration - # to delete old keys prior to the bridge update. - delete_outdated_inbound: false - # What level of device verification should be required from users? - # - # Valid levels: - # unverified - Send keys to all device in the room. - # cross-signed-untrusted - Require valid cross-signing, but trust all cross-signing keys. - # cross-signed-tofu - Require valid cross-signing, trust cross-signing keys on first use (and reject changes). - # cross-signed-verified - Require valid cross-signing, plus a valid user signature from the bridge bot. - # Note that creating user signatures from the bridge bot is not currently possible. - # verified - Require manual per-device verification - # (currently only possible by modifying the `trust` column in the `crypto_device` database table). - verification_levels: - # Minimum level for which the bridge should send keys to when bridging messages from Telegram to Matrix. - receive: unverified - # Minimum level that the bridge should accept for incoming Matrix messages. - send: unverified - # Minimum level that the bridge should require for accepting key requests. - share: cross-signed-tofu - # Options for Megolm room key rotation. These options allow you to - # configure the m.room.encryption event content. See: - # https://spec.matrix.org/v1.3/client-server-api/#mroomencryption for - # more information about that event. - rotation: - # Enable custom Megolm room key rotation settings. Note that these - # settings will only apply to rooms created after this option is - # set. - enable_custom: false - # The maximum number of milliseconds a session should be used - # before changing it. The Matrix spec recommends 604800000 (a week) - # as the default. - milliseconds: 604800000 - # The maximum number of messages that should be sent with a given a - # session before changing it. The Matrix spec recommends 100 as the - # default. - messages: 100 - - # Disable rotating keys when a user's devices change? - # You should not enable this option unless you understand all the implications. - disable_device_change_key_rotation: false - - # Permissions for using the bridge. - # Permitted values: - # relay - Allowed to be relayed through the bridge, no access to commands. - # user - Use the bridge with puppeting. - # admin - Use and administrate the bridge. - # Permitted keys: - # * - All Matrix users - # domain - All users on that homeserver - # mxid - Specific user - permissions: - "*": relay - "{{ MATRIX_SERVER_NAME }}": user - "@{{ MATRIX_ADMINISTRATOR_USERNAME }}:{{ MATRIX_SERVER_NAME }}": admin - - relay: - # Whether relay mode should be allowed. If allowed, `!fb set-relay` can be used to turn any - # authenticated user into a relaybot for that chat. - enabled: false - # The formats to use when sending messages to Messenger via a relay user. - # - # Available variables: - # $sender_displayname - The display name of the sender (e.g. Example User) - # $sender_username - The username (Matrix ID localpart) of the sender (e.g. exampleuser) - # $sender_mxid - The Matrix ID of the sender (e.g. @exampleuser:example.com) - # $message - The message content - message_formats: - m.text: '$sender_displayname: $message' - m.notice: '$sender_displayname: $message' - m.emote: '* $sender_displayname $message' - m.file: '$sender_displayname sent a file' - m.image: '$sender_displayname sent an image' - m.audio: '$sender_displayname sent an audio file' - m.video: '$sender_displayname sent a video' - m.location: '$sender_displayname sent a location' - -facebook: - device_seed: generate - default_region_hint: ODN - connection_type: WIFI - carrier: Verizon - hni: 311390 - - mqtt_keepalive: 60 - -# Python logging configuration. -# -# See section 16.7.2 of the Python documentation for more info: -# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema -logging: - version: 1 - formatters: - colored: - (): mautrix_facebook.util.ColorFormatter - format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" - normal: - format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" - handlers: - file: - class: logging.handlers.RotatingFileHandler - formatter: normal - filename: ./mautrix-facebook.log - maxBytes: 10485760 - backupCount: 10 - console: - class: logging.StreamHandler - formatter: colored - loggers: - mau: - level: DEBUG - maufbapi: - level: DEBUG - paho: - level: INFO - aiohttp: - level: INFO - root: - level: DEBUG - handlers: [file, console] \ No newline at end of file diff --git a/roles/web-app-matrix/templates/flavor/compose/mautrix/instagram.config.yml.j2 b/roles/web-app-matrix/templates/flavor/compose/mautrix/instagram.config.yml.j2 deleted file mode 100644 index 0c7deff7aa..0000000000 --- a/roles/web-app-matrix/templates/flavor/compose/mautrix/instagram.config.yml.j2 +++ /dev/null @@ -1,416 +0,0 @@ -# Homeserver details -homeserver: - # The address that this appservice can use to connect to the homeserver. - address: http://synapse:{{ MATRIX_SYNAPSE_CONTAINER_PORT }} - # The domain of the homeserver (also known as server_name, used for MXIDs, etc). - domain: {{ MATRIX_SERVER_NAME }} - # Whether or not to verify the SSL certificate of the homeserver. - # Only applies if address starts with https:// - verify_ssl: true - # What software is the homeserver running? - # Standard Matrix homeservers like Synapse, Dendrite and Conduit should just use "standard" here. - software: standard - # Number of retries for all HTTP requests if the homeserver isn't reachable. - http_retry_count: 4 - # The URL to push real-time bridge status to. - # If set, the bridge will make POST requests to this URL whenever a user's Instagram MQTT connection state changes. - # The bridge will use the appservice as_token to authorize requests. - status_endpoint: null - # Endpoint for reporting per-message status. - message_send_checkpoint_endpoint: null - # Whether asynchronous uploads via MSC2246 should be enabled for media. - # Requires a media repo that supports MSC2246. - async_media: false - -# Application service host/registration related details -# Changing these values requires regeneration of the registration. -appservice: - # The address that the homeserver can use to connect to this appservice. - address: http://mautrix-instagram:29330 - # When using https:// the TLS certificate and key files for the address. - tls_cert: false - tls_key: false - - # The hostname and port where this appservice should listen. - hostname: 0.0.0.0 - port: 29330 - # The maximum body size of appservice API requests (from the homeserver) in mebibytes - # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s - max_body_size: 1 - - # The full URI to the database. SQLite and Postgres are supported. - # Format examples: - # SQLite: sqlite:filename.db - # Postgres: postgres://username:password@hostname/dbname - database: postgres://mautrix_instagram_bridge:{{ lookup('config', application_id, 'credentials.mautrix_instagram_bridge_database_password')}}@{{ MATRIX_POSTGRES_HOST }}/mautrix_instagram_bridge - # Additional arguments for asyncpg.create_pool() or sqlite3.connect() - # https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool - # https://docs.python.org/3/library/sqlite3.html#sqlite3.connect - # For sqlite, min_size is used as the connection thread pool size and max_size is ignored. - # Additionally, SQLite supports init_commands as an array of SQL queries to run on connect (e.g. to set PRAGMAs). - database_opts: - min_size: 1 - max_size: 10 - - # The unique ID of this appservice. - id: instagram - # Username of the appservice bot. - bot_username: instagrambot - # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty - # to leave display name/avatar as-is. - bot_displayname: Instagram bridge bot - bot_avatar: mxc://maunium.net/JxjlbZUlCPULEeHZSwleUXQv - - # Whether or not to receive ephemeral events via appservice transactions. - # Requires MSC2409 support (i.e. Synapse 1.22+). - # You should disable bridge -> sync_with_custom_puppets when this is enabled. - ephemeral_events: true - - # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify. - as_token: "This value is generated when generating the registration" - hs_token: "This value is generated when generating the registration" - -# Segment-compatible analytics endpoint for tracking some events, like provisioning API login and encryption errors. -analytics: - # Hostname of the tracking server. The path is hardcoded to /v1/track - host: api.segment.io - # API key to send with tracking requests. Tracking is disabled if this is null. - token: null - # Optional user ID for tracking events. If null, defaults to using Matrix user ID. - user_id: null - -# Prometheus telemetry config. Requires prometheus-client to be installed. -metrics: - enabled: false - listen_port: 8000 - -# Manhole config. -manhole: - # Whether or not opening the manhole is allowed. - enabled: false - # The path for the unix socket. - path: /var/tmp/mautrix-instagram.manhole - # The list of UIDs who can be added to the whitelist. - # If empty, any UIDs can be specified in the open-manhole command. - whitelist: - - 0 - -instagram: - # Seed for generating devices. This is secret because the seed is used to generate - # device IDs, which can apparently be used to bypass two-factor authentication after - # logging out, because Instagram is insecure. - device_seed: generate - - mqtt_keepalive: 60 - -# Bridge config -bridge: - # Localpart template of MXIDs for Instagram users. - # {userid} is replaced with the user ID of the Instagram user. - username_template: "instagram_{userid}" - # Displayname template for Instagram users. - # {displayname} is replaced with the display name of the Instagram user. - # {username} is replaced with the username of the Instagram user. - displayname_template: "{displayname} (Instagram)" - # Displayname template for 1:1 chat portals. Same variables as displayname_template. - private_chat_name_template: "{displayname}" - # Displayname template for group chat portals. Only {name} is available. - group_chat_name_template: "{name}" - - # Maximum length of displayname - displayname_max_length: 100 - - # The maximum number of conversations that should be synced when we get a - # message sync error. In general, 1 page (20) is sufficient. - max_startup_thread_sync_count: 20 - # Whether or not to use /sync to get read receipts and typing notifications - # when double puppeting is enabled - sync_with_custom_puppets: false - # Whether or not to update the m.direct account data event when double puppeting is enabled. - # Note that updating the m.direct event is not atomic (except with mautrix-asmux) - # and is therefore prone to race conditions. - sync_direct_chat_list: false - # Allow using double puppeting from any server with a valid client .well-known file. - double_puppet_allow_discovery: false - # Servers to allow double puppeting from, even if double_puppet_allow_discovery is false. - double_puppet_server_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_SYNAPSE_URL }} - # Shared secret for https://github.com/devture/matrix-synapse-shared-secret-auth - # - # If set, custom puppets will be enabled automatically for local users - # instead of users having to find an access token and run `login-matrix` - # manually. - # If using this for other servers than the bridge's server, - # you must also set the URL in the double_puppet_server_map. - login_shared_secret_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_REGISTRATION_SHARED_SECRET }} - # Whether or not created rooms should have federation enabled. - # If false, created portal rooms will never be federated. - federate_rooms: true - # Settings for backfilling messages from Instagram. - backfill: - # Enable initial backfill (~10 messages after creating portal)? - enable_initial: true - # Enable backfill queue? This is used for backfilling additional threads after the initial sync, - # and when MSC2716 is enabled, to backfill message history going backwards. - enable: false - # Use MSC2716 for backfilling? If this is disabled, backfilling only happens when syncing threads, - # and the incremental settings below don't apply. - # - # This requires a server with MSC2716 support, which is currently an experimental feature in Synapse. - # It can be enabled by setting experimental_features -> msc2716_enabled to true in homeserver.yaml. - msc2716: false - # Use double puppets for backfilling? - # In order to use this, the double puppets must be in the appservice's user ID namespace - # (because the bridge can't use the double puppet access token with batch sending). - # This only affects double puppets on the local server, double puppets on other servers will never be used. - double_puppet_backfill: false - # The maximum number of conversations that should be synced. - # Other conversations will be backfilled on demand when the start PM - # provisioning endpoint is used or when a message comes in from that - # chat. - # If set to -1, all conversations will by synced. - max_conversations: 20 - # The minimum amount of time to wait between syncing each thread. This - # helps avoid situations where you sync too quickly. - min_sync_thread_delay: 5 - # If this value is greater than 0, then if the conversation's last - # message was more than this number of hours ago, then the conversation - # will automatically be marked it as read. - # Conversations that have a last message that is less than this number - # of hours ago will have their unread status synced from Instagram. - unread_hours_threshold: 0 - - # Settings for how quickly to backoff when rate-limits are encountered - # while backfilling. - backoff: - # How many seconds to wait after getting rate limited during a - # thread list fetch. - thread_list: 300 - # How many seconds to wait after getting rate limited during a - # message history fetch. - message_history: 300 - - # Settings for backfills. - # - # During initial/incremental sync, the entirety of the thread that is - # available will be backfilled. For example, on initial sync, about 20 - # messages are included for each thread in the thread list returned by - # the server. After that, incremental backfills will be run for each of - # the portals in a round-robin fashion until all portals have been - # backfilled as configured below. - incremental: - # The maximum number of pages to backfill per batch. - max_pages: 10 - # The maximum number of total pages to backfill per portal. - # If set to -1, infinite pages will be synced. - max_total_pages: -1 - # The number of seconds to wait between backfilling each page. - page_delay: 5 - # The number of seconds to wait after backfilling the batch of - # messages. - post_batch_delay: 20 - - periodic_reconnect: - # Interval in seconds in which to automatically reconnect all users. - # This can be used to automatically mitigate the bug where Instagram stops sending messages. - # Set to -1 to disable periodic reconnections entirely. - interval: -1 - # Whether or not the bridge should backfill chats when reconnecting. - resync: true - # Should even disconnected users be reconnected? - always: false - - # URL to call to retrieve a proxy URL from (defaults to the http_proxy environment variable). - get_proxy_api_url: null - # Whether to use proxy for downloading media from Instagram. - use_proxy_for_media: true - - # End-to-bridge encryption support options. - # - # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info. - encryption: - # Allow encryption, work in group chat rooms with e2ee enabled - allow: false - # Default to encryption, force-enable encryption in all portals the bridge creates - # This will cause the bridge bot to be in private chats for the encryption to work properly. - default: false - # Whether to use MSC2409/MSC3202 instead of /sync long polling for receiving encryption-related data. - appservice: false - # Require encryption, drop any unencrypted messages. - require: false - # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled. - # You must use a client that supports requesting keys from other users to use this feature. - allow_key_sharing: false - # Options for deleting megolm sessions from the bridge. - delete_keys: - # Beeper-specific: delete outbound sessions when hungryserv confirms - # that the user has uploaded the key to key backup. - delete_outbound_on_ack: false - # Don't store outbound sessions in the inbound table. - dont_store_outbound: false - # Ratchet megolm sessions forward after decrypting messages. - ratchet_on_decrypt: false - # Delete fully used keys (index >= max_messages) after decrypting messages. - delete_fully_used_on_decrypt: false - # Delete previous megolm sessions from same device when receiving a new one. - delete_prev_on_new_session: false - # Delete megolm sessions received from a device when the device is deleted. - delete_on_device_delete: false - # Periodically delete megolm sessions when 2x max_age has passed since receiving the session. - periodically_delete_expired: false - # Delete inbound megolm sessions that don't have the received_at field used for - # automatic ratcheting and expired session deletion. This is meant as a migration - # to delete old keys prior to the bridge update. - delete_outdated_inbound: false - # What level of device verification should be required from users? - # - # Valid levels: - # unverified - Send keys to all device in the room. - # cross-signed-untrusted - Require valid cross-signing, but trust all cross-signing keys. - # cross-signed-tofu - Require valid cross-signing, trust cross-signing keys on first use (and reject changes). - # cross-signed-verified - Require valid cross-signing, plus a valid user signature from the bridge bot. - # Note that creating user signatures from the bridge bot is not currently possible. - # verified - Require manual per-device verification - # (currently only possible by modifying the `trust` column in the `crypto_device` database table). - verification_levels: - # Minimum level for which the bridge should send keys to when bridging messages from Telegram to Matrix. - receive: unverified - # Minimum level that the bridge should accept for incoming Matrix messages. - send: unverified - # Minimum level that the bridge should require for accepting key requests. - share: cross-signed-tofu - # Options for Megolm room key rotation. These options allow you to - # configure the m.room.encryption event content. See: - # https://spec.matrix.org/v1.3/client-server-api/#mroomencryption for - # more information about that event. - rotation: - # Enable custom Megolm room key rotation settings. Note that these - # settings will only apply to rooms created after this option is - # set. - enable_custom: false - # The maximum number of milliseconds a session should be used - # before changing it. The Matrix spec recommends 604800000 (a week) - # as the default. - milliseconds: 604800000 - # The maximum number of messages that should be sent with a given a - # session before changing it. The Matrix spec recommends 100 as the - # default. - messages: 100 - - # Disable rotating keys when a user's devices change? - # You should not enable this option unless you understand all the implications. - disable_device_change_key_rotation: false - - # Whether to explicitly set the avatar and room name for private chat portal rooms. - # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms. - # If set to `always`, all DM rooms will have explicit names and avatars set. - # If set to `never`, DM rooms will never have names and avatars set. - private_chat_portal_meta: default - # Whether or not the bridge should send a read receipt from the bridge bot when a message has - # been sent to Instagram. - delivery_receipts: false - # Whether or not delivery errors should be reported as messages in the Matrix room. - delivery_error_reports: false - # Whether the bridge should send the message status as a custom com.beeper.message_send_status event. - message_status_events: false - # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run. - # This field will automatically be changed back to false after it, - # except if the config file is not writable. - resend_bridge_info: false - # Whether or not unimportant bridge notices should be sent to the user. - # (e.g. connected, disconnected but will retry) - unimportant_bridge_notices: true - # Disable bridge notices entirely - disable_bridge_notices: false - # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552. - # This is currently not supported in most clients. - caption_in_message: false - - # Should Matrix m.notice-type messages be bridged? - bridge_notices: true - - # Should Matrix typing notices be sent to Instagram? Typing notifications - # from Instagram will still be bridged. - bridge_matrix_typing: true - - # Provisioning API part of the web server for automated portal creation and fetching information. - # Used by things like mautrix-manager (https://github.com/tulir/mautrix-manager). - provisioning: - # Whether or not the provisioning API should be enabled. - enabled: true - # The prefix to use in the provisioning API endpoints. - prefix: /_matrix/provision/v1 - # The shared secret to authorize users of the API. - # Set to "generate" to generate and save a new token. - shared_secret: generate - - # The prefix for commands. Only required in non-management rooms. - command_prefix: "!ig" - - # Permissions for using the bridge. - # Permitted values: - # relay - Allowed to be relayed through the bridge, no access to commands. - # user - Use the bridge with puppeting. - # admin - Use and administrate the bridge. - # Permitted keys: - # * - All Matrix users - # domain - All users on that homeserver - # mxid - Specific user - permissions: - "*": relay - "{{ MATRIX_SERVER_NAME }}": user - "@{{ MATRIX_ADMINISTRATOR_USERNAME }}:{{ MATRIX_SERVER_NAME }}": admin - - relay: - # Whether relay mode should be allowed. If allowed, `!ig set-relay` can be used to turn any - # authenticated user into a relaybot for that chat. - enabled: false - # The formats to use when sending messages to Instagram via a relay user. - # - # Available variables: - # $sender_displayname - The display name of the sender (e.g. Example User) - # $sender_username - The username (Matrix ID localpart) of the sender (e.g. exampleuser) - # $sender_mxid - The Matrix ID of the sender (e.g. @exampleuser:example.com) - # $message - The message content - # - # Note that Instagram doesn't support captions for images, so images won't include any indication of being relayed. - message_formats: - m.text: '$sender_displayname: $message' - m.notice: '$sender_displayname: $message' - m.emote: '* $sender_displayname $message' - -# Python logging configuration. -# -# See section 16.7.2 of the Python documentation for more info: -# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema -logging: - version: 1 - formatters: - colored: - (): mautrix_instagram.util.ColorFormatter - format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" - normal: - format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" - handlers: - file: - class: logging.handlers.RotatingFileHandler - formatter: normal - filename: ./mautrix-instagram.log - maxBytes: 10485760 - backupCount: 10 - console: - class: logging.StreamHandler - formatter: colored - loggers: - mau: - level: DEBUG - mauigpapi: - level: DEBUG - aiohttp: - level: INFO - paho.mqtt: - level: INFO - root: - level: DEBUG - handlers: [file, console] \ No newline at end of file diff --git a/roles/web-app-matrix/templates/flavor/compose/mautrix/meta.config.yml.j2 b/roles/web-app-matrix/templates/flavor/compose/mautrix/meta.config.yml.j2 new file mode 100644 index 0000000000..caab6213a0 --- /dev/null +++ b/roles/web-app-matrix/templates/flavor/compose/mautrix/meta.config.yml.j2 @@ -0,0 +1,233 @@ + +network: + mode: + allow_messenger_com_on_fb: false + allowed_modes: [] + +{% raw %} + displayname_template: '{{or .DisplayName .Username "Unknown user"}}' +{% endraw %} + + proxy: + get_proxy_from: + proxy_media: false + proxy_e2ee: false + proxy_messenger_lite: true + proxy_other: true + min_full_reconnect_interval_seconds: 3600 + force_refresh_interval_seconds: 72000 + cache_connection_state: false + disable_xma_backfill: true + disable_xma_always: false + send_presence_on_typing: false + receive_instagram_typing_indicators: true + disable_view_once: false + marketplace_space: true + log_redacted_bloks_payloads: false + + thread_backfill: + batch_count: -1 + batch_delay: 2s + +bridge: + command_prefix: "!meta" + personal_filtering_spaces: false + private_chat_portal_meta: true + async_events: false + split_portals: false + resend_bridge_info: false + no_bridge_info_state_key: false + bridge_status_notices: errors + unknown_error_auto_reconnect: null + unknown_error_max_auto_reconnects: 10 + + bridge_matrix_leave: true + bridge_notices: true + tag_only_on_create: true + only_bridge_tags: [m.favourite, m.lowpriority] + mute_only_on_create: true + deduplicate_matrix_messages: false + cross_room_replies: false + revert_failed_state_changes: false + kick_matrix_users: true + enable_send_state_requests: false + + cleanup_on_logout: + enabled: false + manual: + private: nothing + relayed: nothing + shared_no_users: nothing + shared_has_users: nothing + bad_credentials: + private: nothing + relayed: nothing + shared_no_users: nothing + shared_has_users: nothing + + relay: + enabled: false + admin_only: true + prefer_default: true + allow_bridge: true + default_relays: [] + message_formats: +{% raw %} + m.text: "{{ .Sender.DisambiguatedName }}: {{ .Message }}" + m.notice: "{{ .Sender.DisambiguatedName }}: {{ .Message }}" + m.emote: "* {{ .Sender.DisambiguatedName }} {{ .Message }}" + m.file: "{{ .Sender.DisambiguatedName }} sent a file{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.image: "{{ .Sender.DisambiguatedName }} sent an image{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.audio: "{{ .Sender.DisambiguatedName }} sent an audio file{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.video: "{{ .Sender.DisambiguatedName }} sent a video{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.location: "{{ .Sender.DisambiguatedName }} sent a location{{ if .Caption }}: {{ .Caption }}{{ end }}" +{% endraw %} + + portal_create_filter: + mode: deny + list: [] + always_deny_from_login: [] + + permissions: + "*": relay + "{{ MATRIX_SERVER_NAME }}": user + "@{{ MATRIX_ADMINISTRATOR_USERNAME }}:{{ MATRIX_SERVER_NAME }}": admin + +database: + type: postgres + uri: postgres://mautrix_meta_bridge:{{ lookup('config', application_id, 'credentials.mautrix_meta_bridge_database_password') | regex_replace('/', '%2F') | regex_replace('@', '%40') | regex_replace(':', '%3A') }}@{{ MATRIX_POSTGRES_HOST }}/mautrix_meta_bridge?sslmode=disable + max_open_conns: 20 + max_idle_conns: 2 + max_conn_idle_time: null + max_conn_lifetime: null + +homeserver: + address: http://synapse:{{ MATRIX_SYNAPSE_CONTAINER_PORT }} + domain: {{ MATRIX_SERVER_NAME }} + + software: standard + status_endpoint: + message_send_checkpoint_endpoint: + async_media: false + + websocket: false + ping_interval_seconds: 0 + retry_limit: 4 + +appservice: + address: http://mautrix-meta:29319 + + hostname: 0.0.0.0 + port: 29319 + + id: meta + bot: + username: metabot + displayname: Meta bridge bot + avatar: mxc://maunium.net/DxpVrwwzPUwaUSazpsjXgcKB + + ephemeral_events: true + async_transactions: false + + as_token: "This value is generated when generating the registration" + hs_token: "This value is generated when generating the registration" + +{% raw %} + username_template: meta_{{.}} +{% endraw %} + +matrix: + message_status_events: false + delivery_receipts: false + message_error_notices: true + sync_direct_chat_list: false + federate_rooms: true + upload_file_threshold: 5242880 + ghost_extra_profile_info: false + +analytics: + token: null + url: null + user_id: null + +provisioning: + shared_secret: generate + allow_matrix_auth: true + debug_endpoints: false + enable_session_transfers: false + +public_media: + enabled: false + signing_key: generate + expiry: 0 + hash_length: 32 + path_prefix: /_mautrix/publicmedia + use_database: false + +direct_media: + enabled: false + server_name: meta-media.example.com + well_known_response: + media_id_prefix: + allow_proxy: true + server_key: generate + +backfill: + enabled: true + max_initial_messages: 50 + max_catchup_messages: 500 + unread_hours_threshold: 720 + threads: + max_initial_messages: 50 + queue: + enabled: false + manual: false + batch_size: 100 + batch_delay: 20 + max_batches: -1 + max_batches_override: {} + +double_puppet: + servers: + {{ MATRIX_SERVER_NAME }}: {{ MATRIX_SYNAPSE_URL }} + allow_discovery: false + secrets: + {{ MATRIX_SERVER_NAME }}: {{ MATRIX_REGISTRATION_SHARED_SECRET }} + +encryption: + allow: false + default: false + require: false + appservice: false + msc4190: false + msc4392: false + self_sign: false + allow_key_sharing: false + plaintext_mentions: false + pickle_key: generate + delete_keys: + delete_outbound_on_ack: false + dont_store_outbound: false + ratchet_on_decrypt: false + delete_fully_used_on_decrypt: false + delete_prev_on_new_session: false + delete_on_device_delete: false + periodically_delete_expired: false + delete_outdated_inbound: false + verification_levels: + receive: unverified + send: unverified + share: cross-signed-tofu + rotation: + enable_custom: false + milliseconds: 604800000 + messages: 100 + disable_device_change_key_rotation: false + +env_config_prefix: null + +logging: + min_level: debug + writers: + - type: stdout + format: pretty-colored diff --git a/roles/web-app-matrix/templates/flavor/compose/mautrix/signal.config.yml.j2 b/roles/web-app-matrix/templates/flavor/compose/mautrix/signal.config.yml.j2 index d3482328b4..6d7798eff0 100644 --- a/roles/web-app-matrix/templates/flavor/compose/mautrix/signal.config.yml.j2 +++ b/roles/web-app-matrix/templates/flavor/compose/mautrix/signal.config.yml.j2 @@ -1,310 +1,133 @@ -# Homeserver details. -homeserver: - # The address that this appservice can use to connect to the homeserver. - address: http://synapse:{{ MATRIX_SYNAPSE_CONTAINER_PORT }} - # The domain of the homeserver (also known as server_name, used for MXIDs, etc). - domain: {{ MATRIX_SERVER_NAME }} - - # What software is the homeserver running? - # Standard Matrix homeservers like Synapse, Dendrite and Conduit should just use "standard" here. - software: standard - # The URL to push real-time bridge status to. - # If set, the bridge will make POST requests to this URL whenever a user's discord connection state changes. - # The bridge will use the appservice as_token to authorize requests. - status_endpoint: null - # Endpoint for reporting per-message status. - message_send_checkpoint_endpoint: null - # Does the homeserver support https://github.com/matrix-org/matrix-spec-proposals/pull/2246? - async_media: false - - # Should the bridge use a websocket for connecting to the homeserver? - # The server side is currently not documented anywhere and is only implemented by mautrix-wsproxy, - # mautrix-asmux (deprecated), and hungryserv (proprietary). - websocket: false - # How often should the websocket be pinged? Pinging will be disabled if this is zero. - ping_interval_seconds: 0 - -# Application service host/registration related details. -# Changing these values requires regeneration of the registration. -appservice: - # The address that the homeserver can use to connect to this appservice. - address: http://mautrix-signal:29328 - - # The hostname and port where this appservice should listen. - hostname: 0.0.0.0 - port: 29328 - - # Database config. - database: - # The database type. "sqlite3-fk-wal" and "postgres" are supported. - type: postgres - # The database URI. - # SQLite: A raw file path is supported, but `file:?_txlock=immediate` is recommended. - # https://github.com/mattn/go-sqlite3#connection-string - # Postgres: Connection string. For example, postgres://user:password@host/database?sslmode=disable - # To connect via Unix socket, use something like postgres:///dbname?host=/var/run/postgresql - uri: postgres://mautrix_signal_bridge:{{ lookup('config', application_id, 'credentials.mautrix_signal_bridge_database_password')}}@{{ MATRIX_POSTGRES_HOST }}/mautrix_signal_bridge?sslmode=disable - # Maximum number of connections. Mostly relevant for Postgres. - max_open_conns: 20 - max_idle_conns: 2 - # Maximum connection idle time and lifetime before they're closed. Disabled if null. - # Parsed with https://pkg.go.dev/time#ParseDuration - max_conn_idle_time: null - max_conn_lifetime: null - - # The unique ID of this appservice. - id: signal - # Appservice bot details. - bot: - # Username of the appservice bot. - username: signalbot - # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty - # to leave display name/avatar as-is. - displayname: Signal bridge bot - avatar: mxc://maunium.net/wPJgTQbZOtpBFmDNkiNEMDUp - - # Whether or not to receive ephemeral events via appservice transactions. - # Requires MSC2409 support (i.e. Synapse 1.22+). - ephemeral_events: true - - # Should incoming events be handled asynchronously? - # This may be necessary for large public instances with lots of messages going through. - # However, messages will not be guaranteed to be bridged in the same order they were sent in. - async_transactions: false - - # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify. - as_token: "This value is generated when generating the registration" - hs_token: "This value is generated when generating the registration" - -# Prometheus config. -metrics: - # Enable prometheus metrics? - enabled: false - # IP and port where the metrics listener should be. The path is always /metrics - listen: 127.0.0.1:8000 - -signal: - # Default device name that shows up in the Signal app. - device_name: mautrix-signal - -# Bridge config bridge: -{% raw %} - # Localpart template of MXIDs for Signal users. - # {{.}} is replaced with the internal ID of the Signal user. - username_template: signal_{{.}} - # Displayname template for Signal users. This is also used as the room name in DMs if private_chat_portal_meta is enabled. - # {{.ProfileName}} - The Signal profile name set by the user. - # {{.ContactName}} - The name for the user from your phone's contact list. This is not safe on multi-user instances. - # {{.PhoneNumber}} - The phone number of the user. - # {{.UUID}} - The UUID of the Signal user. - # {{.AboutEmoji}} - The emoji set by the user in their profile. - displayname_template: '{{or .ProfileName .PhoneNumber "Unknown user"}}' -{% endraw %} - # Whether to explicitly set the avatar and room name for private chat portal rooms. - # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms. - # If set to `always`, all DM rooms will have explicit names and avatars set. - # If set to `never`, DM rooms will never have names and avatars set. - private_chat_portal_meta: default - # Should avatars from the user's contact list be used? This is not safe on multi-user instances. - use_contact_avatars: false - # Should the Signal user's phone number be included in the room topic in private chat portal rooms? - number_in_topic: true - # Avatar image for the Note to Self room. - note_to_self_avatar: mxc://maunium.net/REBIVrqjZwmaWpssCZpBlmlL - - portal_message_buffer: 128 - - # Should the bridge create a space for each logged-in user and add bridged rooms to it? - # Users who logged in before turning this on should run `!signal sync-space` to create and fill the space for the first time. - personal_filtering_spaces: false - # Should Matrix m.notice-type messages be bridged? - bridge_notices: true - # Should the bridge send a read receipt from the bridge bot when a message has been sent to Signal? - delivery_receipts: false - # Whether the bridge should send the message status as a custom com.beeper.message_send_status event. - message_status_events: false - # Whether the bridge should send error notices via m.notice events when a message fails to bridge. - message_error_notices: true - # Should the bridge update the m.direct account data event when double puppeting is enabled. - # Note that updating the m.direct event is not atomic (except with mautrix-asmux) - # and is therefore prone to race conditions. - sync_direct_chat_list: false - # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run. - # This field will automatically be changed back to false after it, except if the config file is not writable. - resend_bridge_info: false - # Send captions in the same message as images. This will send data compatible with both MSC2530. - # This is currently not supported in most clients. - caption_in_message: false - # Whether or not created rooms should have federation enabled. - # If false, created portal rooms will never be federated. - federate_rooms: true - # Servers to always allow double puppeting from - double_puppet_server_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_SYNAPSE_URL }} - # Allow using double puppeting from any server with a valid client .well-known file. - double_puppet_allow_discovery: false - # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth - # - # If set, double puppeting will be enabled automatically for local users - # instead of users having to find an access token and run `login-matrix` - # manually. - login_shared_secret_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_REGISTRATION_SHARED_SECRET }} - - # Maximum time for handling Matrix events. Duration strings formatted for https://pkg.go.dev/time#ParseDuration - # Null means there's no enforced timeout. - message_handling_timeout: - # Send an error message after this timeout, but keep waiting for the response until the deadline. - # This is counted from the origin_server_ts, so the warning time is consistent regardless of the source of delay. - # If the message is older than this when it reaches the bridge, the message won't be handled at all. - error_after: null - # Drop messages after this timeout. They may still go through if the message got sent to the servers. - # This is counted from the time the bridge starts handling the message. - deadline: 120s - - # The prefix for commands. Only required in non-management rooms. command_prefix: '!signal' - # Messages sent upon joining a management room. - # Markdown is supported. The defaults are listed below. - management_room_text: - # Sent when joining a room. - welcome: "Hello, I'm a Signal bridge bot." - # Sent when joining a management room and the user is already logged in. - welcome_connected: "Use `help` for help." - # Sent when joining a management room and the user is not logged in. - welcome_unconnected: "Use `help` for help or `login` to log in." - # Optional extra text sent when joining a management room. - additional_help: "" + personal_filtering_spaces: false + private_chat_portal_meta: false + bridge_notices: true + bridge_status_notices: errors - # End-to-bridge encryption support options. - # - # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info. - encryption: - # Allow encryption, work in group chat rooms with e2ee enabled - allow: false - # Default to encryption, force-enable encryption in all portals the bridge creates - # This will cause the bridge bot to be in private chats for the encryption to work properly. - default: false - # Whether to use MSC2409/MSC3202 instead of /sync long polling for receiving encryption-related data. - appservice: false - # Require encryption, drop any unencrypted messages. - require: false - # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled. - # You must use a client that supports requesting keys from other users to use this feature. - allow_key_sharing: false - # Options for deleting megolm sessions from the bridge. - delete_keys: - # Beeper-specific: delete outbound sessions when hungryserv confirms - # that the user has uploaded the key to key backup. - delete_outbound_on_ack: false - # Don't store outbound sessions in the inbound table. - dont_store_outbound: false - # Ratchet megolm sessions forward after decrypting messages. - ratchet_on_decrypt: false - # Delete fully used keys (index >= max_messages) after decrypting messages. - delete_fully_used_on_decrypt: false - # Delete previous megolm sessions from same device when receiving a new one. - delete_prev_on_new_session: false - # Delete megolm sessions received from a device when the device is deleted. - delete_on_device_delete: false - # Periodically delete megolm sessions when 2x max_age has passed since receiving the session. - periodically_delete_expired: false - # Delete inbound megolm sessions that don't have the received_at field used for - # automatic ratcheting and expired session deletion. This is meant as a migration - # to delete old keys prior to the bridge update. - delete_outdated_inbound: false - # What level of device verification should be required from users? - # - # Valid levels: - # unverified - Send keys to all device in the room. - # cross-signed-untrusted - Require valid cross-signing, but trust all cross-signing keys. - # cross-signed-tofu - Require valid cross-signing, trust cross-signing keys on first use (and reject changes). - # cross-signed-verified - Require valid cross-signing, plus a valid user signature from the bridge bot. - # Note that creating user signatures from the bridge bot is not currently possible. - # verified - Require manual per-device verification - # (currently only possible by modifying the `trust` column in the `crypto_device` database table). - verification_levels: - # Minimum level for which the bridge should send keys to when bridging messages from Signal to Matrix. - receive: unverified - # Minimum level that the bridge should accept for incoming Matrix messages. - send: unverified - # Minimum level that the bridge should require for accepting key requests. - share: cross-signed-tofu - # Options for Megolm room key rotation. These options allow you to - # configure the m.room.encryption event content. See: - # https://spec.matrix.org/v1.3/client-server-api/#mroomencryption for - # more information about that event. - rotation: - # Enable custom Megolm room key rotation settings. Note that these - # settings will only apply to rooms created after this option is - # set. - enable_custom: false - # The maximum number of milliseconds a session should be used - # before changing it. The Matrix spec recommends 604800000 (a week) - # as the default. - milliseconds: 604800000 - # The maximum number of messages that should be sent with a given a - # session before changing it. The Matrix spec recommends 100 as the - # default. - messages: 100 + relay: + enabled: false + admin_only: true + message_formats: +{% raw %} + m.text: "{{ .Sender.DisambiguatedName }}: {{ .Message }}" + m.notice: "{{ .Sender.DisambiguatedName }}: {{ .Message }}" + m.emote: "* {{ .Sender.DisambiguatedName }} {{ .Message }}" + m.file: "{{ .Sender.DisambiguatedName }} sent a file{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.image: "{{ .Sender.DisambiguatedName }} sent an image{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.audio: "{{ .Sender.DisambiguatedName }} sent an audio file{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.video: "{{ .Sender.DisambiguatedName }} sent a video{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.location: "{{ .Sender.DisambiguatedName }} sent a location{{ if .Caption }}: {{ .Caption }}{{ end }}" +{% endraw %} - # Disable rotating keys when a user's devices change? - # You should not enable this option unless you understand all the implications. - disable_device_change_key_rotation: false - - # Settings for provisioning API - provisioning: - # Prefix for the provisioning API paths. - prefix: /_matrix/provision - # Shared secret for authentication. If set to "generate", a random secret will be generated, - # or if set to "disable", the provisioning API will be disabled. - shared_secret: generate - # Enable debug API at /debug with provisioning authentication. - debug_endpoints: false - - # Permissions for using the bridge. - # Permitted values: - # relay - Talk through the relaybot (if enabled), no access otherwise - # user - Access to use the bridge to chat with a Signal account. - # admin - User level and some additional administration tools - # Permitted keys: - # * - All Matrix users - # domain - All users on that homeserver - # mxid - Specific user permissions: "*": relay "{{ MATRIX_SERVER_NAME }}": user "@{{ MATRIX_ADMINISTRATOR_USERNAME }}:{{ MATRIX_SERVER_NAME }}": admin - # Settings for relay mode - relay: - # Whether relay mode should be allowed. If allowed, `!wa set-relay` can be used to turn any - # authenticated user into a relaybot for that chat. - enabled: false - # Should only admins be allowed to set themselves as relay users? - admin_only: true - # The formats to use when sending messages to Signal via the relaybot. - message_formats: +database: + type: postgres + uri: postgres://mautrix_signal_bridge:{{ lookup('config', application_id, 'credentials.mautrix_signal_bridge_database_password') | regex_replace('/', '%2F') | regex_replace('@', '%40') | regex_replace(':', '%3A') }}@{{ MATRIX_POSTGRES_HOST }}/mautrix_signal_bridge?sslmode=disable + max_open_conns: 20 + max_idle_conns: 2 + max_conn_idle_time: null + max_conn_lifetime: null + +homeserver: + address: http://synapse:{{ MATRIX_SYNAPSE_CONTAINER_PORT }} + domain: {{ MATRIX_SERVER_NAME }} + + software: standard + status_endpoint: + message_send_checkpoint_endpoint: + async_media: false + + websocket: false + ping_interval_seconds: 0 + retry_limit: 4 + +appservice: + address: http://mautrix-signal:29328 + + hostname: 0.0.0.0 + port: 29328 + + id: signal + bot: + username: signalbot + displayname: Signal bridge bot + avatar: mxc://maunium.net/wPJgTQbZOtpBFmDNkiNEMDUp + + ephemeral_events: true + async_transactions: false + + as_token: "This value is generated when generating the registration" + hs_token: "This value is generated when generating the registration" + {% raw %} - m.text: "{{ .Sender.Displayname }}: {{ .Message }}" - m.notice: "{{ .Sender.Displayname }}: {{ .Message }}" - m.emote: "* {{ .Sender.Displayname }} {{ .Message }}" - m.file: "{{ .Sender.Displayname }} sent a file" - m.image: "{{ .Sender.Displayname }} sent an image" - m.audio: "{{ .Sender.Displayname }} sent an audio file" - m.video: "{{ .Sender.Displayname }} sent a video" - m.location: "{{ .Sender.Displayname }} sent a location" + username_template: signal_{{.}} {% endraw %} -# Logging config. See https://github.com/tulir/zeroconfig for details. + +matrix: + message_status_events: false + delivery_receipts: false + message_error_notices: true + sync_direct_chat_list: false + federate_rooms: true + +provisioning: + shared_secret: generate + debug_endpoints: false + +backfill: + enabled: false + +double_puppet: + servers: + {{ MATRIX_SERVER_NAME }}: {{ MATRIX_SYNAPSE_URL }} + allow_discovery: false + secrets: + {{ MATRIX_SERVER_NAME }}: {{ MATRIX_REGISTRATION_SHARED_SECRET }} + +encryption: + allow: false + default: false + require: false + appservice: false + allow_key_sharing: false + pickle_key: generate + verification_levels: + receive: unverified + send: unverified + share: cross-signed-tofu + rotation: + enable_custom: false + milliseconds: 604800000 + messages: 100 + disable_device_change_key_rotation: false + +network: +{% raw %} + displayname_template: '{{or .ProfileName .PhoneNumber "Unknown user"}}' +{% endraw %} + use_contact_avatars: false + sync_contacts_on_startup: true + use_outdated_profiles: false + number_in_topic: true + device_name: mautrix-signal + note_to_self_avatar: mxc://maunium.net/REBIVrqjZwmaWpssCZpBlmlL + location_format: 'https://www.google.com/maps/place/%[1]s,%[2]s' + disappear_view_once: false + extev_polls: false + logging: min_level: debug writers: - - type: stdout - format: pretty-colored - - type: file - format: json - filename: ./logs/mautrix-signal.log - max_size: 100 - max_backups: 10 - compress: true \ No newline at end of file + - type: stdout + format: pretty-colored + - type: file + format: json + filename: ./logs/mautrix-signal.log + max_size: 100 + max_backups: 10 + compress: true diff --git a/roles/web-app-matrix/templates/flavor/compose/mautrix/slack.config.yml.j2 b/roles/web-app-matrix/templates/flavor/compose/mautrix/slack.config.yml.j2 index 1952ed557d..ff4eebe5b4 100644 --- a/roles/web-app-matrix/templates/flavor/compose/mautrix/slack.config.yml.j2 +++ b/roles/web-app-matrix/templates/flavor/compose/mautrix/slack.config.yml.j2 @@ -1,295 +1,136 @@ -# Homeserver details. + +database: + type: postgres + uri: postgres://mautrix_slack_bridge:{{ lookup('config', application_id, 'credentials.mautrix_slack_bridge_database_password') | regex_replace('/', '%2F') | regex_replace('@', '%40') | regex_replace(':', '%3A') }}@{{ MATRIX_POSTGRES_HOST }}/mautrix_slack_bridge?sslmode=disable + max_open_conns: 20 + max_idle_conns: 2 + max_conn_idle_time: null + max_conn_lifetime: null + homeserver: - # The address that this appservice can use to connect to the homeserver. address: http://synapse:{{ MATRIX_SYNAPSE_CONTAINER_PORT }} - # The domain of the homeserver (also known as server_name, used for MXIDs, etc). domain: {{ MATRIX_SERVER_NAME }} - # What software is the homeserver running? - # Standard Matrix homeservers like Synapse, Dendrite and Conduit should just use "standard" here. software: standard - # The URL to push real-time bridge status to. - # If set, the bridge will make POST requests to this URL whenever a user's slack connection state changes. - # The bridge will use the appservice as_token to authorize requests. status_endpoint: null - # Endpoint for reporting per-message status. message_send_checkpoint_endpoint: null - # Does the homeserver support https://github.com/matrix-org/matrix-spec-proposals/pull/2246? async_media: false - # Should the bridge use a websocket for connecting to the homeserver? - # The server side is currently not documented anywhere and is only implemented by mautrix-wsproxy, - # mautrix-asmux (deprecated), and hungryserv (proprietary). websocket: false - # How often should the websocket be pinged? Pinging will be disabled if this is zero. ping_interval_seconds: 0 + retry_limit: 4 -# Application service host/registration related details. -# Changing these values requires regeneration of the registration. appservice: - # The address that the homeserver can use to connect to this appservice. address: http://mautrix-slack:29335 - # The hostname and port where this appservice should listen. hostname: 0.0.0.0 port: 29335 - # Database config. - database: - # The database type. "sqlite3-fk-wal" and "postgres" are supported. - type: postgres - # The database URI. - # SQLite: A raw file path is supported, but `file:?_txlock=immediate` is recommended. - # https://github.com/mattn/go-sqlite3#connection-string - # Postgres: Connection string. For example, postgres://user:password@host/database?sslmode=disable - # To connect via Unix socket, use something like postgres:///dbname?host=/var/run/postgresql - uri: postgres://mautrix_slack_bridge:{{ lookup('config', application_id, 'credentials.mautrix_slack_bridge_database_password')}}@{{ MATRIX_POSTGRES_HOST }}/mautrix_slack_bridge?sslmode=disable - # Maximum number of connections. Mostly relevant for Postgres. - max_open_conns: 20 - max_idle_conns: 2 - # Maximum connection idle time and lifetime before they're closed. Disabled if null. - # Parsed with https://pkg.go.dev/time#ParseDuration - max_conn_idle_time: null - max_conn_lifetime: null - - # The unique ID of this appservice. id: slack - # Appservice bot details. bot: - # Username of the appservice bot. username: slackbot - # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty - # to leave display name/avatar as-is. displayname: Slack bridge bot avatar: mxc://maunium.net/pVtzLmChZejGxLqmXtQjFxem - # Whether or not to receive ephemeral events via appservice transactions. - # Requires MSC2409 support (i.e. Synapse 1.22+). - # You should disable bridge -> sync_with_custom_puppets when this is enabled. ephemeral_events: true - - # Should incoming events be handled asynchronously? - # This may be necessary for large public instances with lots of messages going through. - # However, messages will not be guaranteed to be bridged in the same order they were sent in. async_transactions: false - # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify. as_token: "This value is generated when generating the registration" hs_token: "This value is generated when generating the registration" -# Bridge config -bridge: {% raw %} - # Localpart template of MXIDs for Slack users. - # {{.}} is replaced with the internal ID of the Slack user. username_template: slack_{{.}} - # Displayname template for Slack users. - # TODO: document variables - displayname_template: '{{.RealName}} (S)' - bot_displayname_template: '{{.Name}} (bot)' - channel_name_template: '#{{.Name}}' {% endraw %} - portal_message_buffer: 128 - # Should the bridge send a read receipt from the bridge bot when a message has been sent to Slack? - delivery_receipts: true - # Whether the bridge should send the message status as a custom com.beeper.message_send_status event. - message_status_events: false - # Whether the bridge should send error notices via m.notice events when a message fails to bridge. - message_error_notices: true - # Should incoming custom emoji reactions be bridged as mxc:// URIs? - # If set to false, custom emoji reactions will be bridged as the shortcode instead, and the image won't be available. - custom_emoji_reactions: true - - # Should the bridge sync with double puppeting to receive EDUs that aren't normally sent to appservices. - sync_with_custom_puppets: false - # Should the bridge update the m.direct account data event when double puppeting is enabled. - # Note that updating the m.direct event is not atomic (except with mautrix-asmux) - # and is therefore prone to race conditions. - sync_direct_chat_list: false - # Whether or not created rooms should have federation enabled. - # If false, created portal rooms will never be federated. - federate_rooms: true - # Whether to explicitly set the avatar and room name for private chat portal rooms. - # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms. - # If set to `always`, all DM rooms will have explicit names and avatars set. - # If set to `never`, DM rooms will never have names and avatars set. - private_chat_portal_meta: default - - # Servers to always allow double puppeting from - double_puppet_server_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_SYNAPSE_URL }} - # Allow using double puppeting from any server with a valid client .well-known file. - double_puppet_allow_discovery: false - # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth - # - # If set, double puppeting will be enabled automatically for local users - # instead of users having to find an access token and run `login-matrix` - # manually. - login_shared_secret_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_REGISTRATION_SHARED_SECRET }} - - message_handling_timeout: - # Send an error message after this timeout, but keep waiting for the response until the deadline. - # This is counted from the origin_server_ts, so the warning time is consistent regardless of the source of delay. - # If the message is older than this when it reaches the bridge, the message won't be handled at all. - error_after: 10s - # Drop messages after this timeout. They may still go through if the message got sent to the servers. - # This is counted from the time the bridge starts handling the message. - deadline: 60s - - # The prefix for commands. Only required in non-management rooms. +bridge: command_prefix: '!slack' - # Messages sent upon joining a management room. - # Markdown is supported. The defaults are listed below. - management_room_text: - # Sent when joining a room. - welcome: "Hello, I'm a Slack bridge bot." - # Sent when joining a management room and the user is already logged in. - welcome_connected: "Use `help` for help." - # Sent when joining a management room and the user is not logged in. - welcome_unconnected: "Use `help` for help, or `login-token` or `login-password` to log in." - # Optional extra text sent when joining a management room. - additional_help: "" + personal_filtering_spaces: true + private_chat_portal_meta: true + bridge_status_notices: errors + relay: + enabled: false + admin_only: true + default_relays: [] + message_formats: +{% raw %} + m.text: "{{ .Sender.DisambiguatedName }}: {{ .Message }}" + m.notice: "{{ .Sender.DisambiguatedName }}: {{ .Message }}" + m.emote: "* {{ .Sender.DisambiguatedName }} {{ .Message }}" +{% endraw %} - backfill: - # Allow backfilling at all? Requires MSC2716 support on homeserver. - enable: false - - # Maximum number of conversations to fetch from Slack when syncing team from Slack. - # Must be 0-999 - conversations_count: 200 - - # If a backfilled chat is older than this number of hours, mark it as read even if it's unread on Slack. - # Set to -1 to let any chat be unread. - unread_hours_threshold: 720 - - # Number of messages to immediately backfill when creating a portal. - immediate_messages: 10 - - # Settings for incremental backfill of history. - incremental: - # Maximum number of messages to backfill per batch. - messages_per_batch: 100 - # The number of seconds to wait after backfilling the batch of messages. - post_batch_delay: 20 - # The maximum number of messages to backfill per portal, split by the chat type. - # If set to -1, all messages in the chat will eventually be backfilled. - max_messages: - # Channels - channel: -1 - # Group direct messages - group_dm: -1 - # 1:1 direct messages - dm: -1 - - # End-to-bridge encryption support options. - # - # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info. - encryption: - # Allow encryption, work in group chat rooms with e2ee enabled - allow: false - # Default to encryption, force-enable encryption in all portals the bridge creates - # This will cause the bridge bot to be in private chats for the encryption to work properly. - default: false - # Whether to use MSC2409/MSC3202 instead of /sync long polling for receiving encryption-related data. - appservice: false - # Require encryption, drop any unencrypted messages. - require: false - # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled. - # You must use a client that supports requesting keys from other users to use this feature. - allow_key_sharing: false - # Options for deleting megolm sessions from the bridge. - delete_keys: - # Beeper-specific: delete outbound sessions when hungryserv confirms - # that the user has uploaded the key to key backup. - delete_outbound_on_ack: false - # Don't store outbound sessions in the inbound table. - dont_store_outbound: false - # Ratchet megolm sessions forward after decrypting messages. - ratchet_on_decrypt: false - # Delete fully used keys (index >= max_messages) after decrypting messages. - delete_fully_used_on_decrypt: false - # Delete previous megolm sessions from same device when receiving a new one. - delete_prev_on_new_session: false - # Delete megolm sessions received from a device when the device is deleted. - delete_on_device_delete: false - # Periodically delete megolm sessions when 2x max_age has passed since receiving the session. - periodically_delete_expired: false - # Delete inbound megolm sessions that don't have the received_at field used for - # automatic ratcheting and expired session deletion. This is meant as a migration - # to delete old keys prior to the bridge update. - delete_outdated_inbound: false - # What level of device verification should be required from users? - # - # Valid levels: - # unverified - Send keys to all device in the room. - # cross-signed-untrusted - Require valid cross-signing, but trust all cross-signing keys. - # cross-signed-tofu - Require valid cross-signing, trust cross-signing keys on first use (and reject changes). - # cross-signed-verified - Require valid cross-signing, plus a valid user signature from the bridge bot. - # Note that creating user signatures from the bridge bot is not currently possible. - # verified - Require manual per-device verification - # (currently only possible by modifying the `trust` column in the `crypto_device` database table). - verification_levels: - # Minimum level for which the bridge should send keys to when bridging messages from WhatsApp to Matrix. - receive: unverified - # Minimum level that the bridge should accept for incoming Matrix messages. - send: unverified - # Minimum level that the bridge should require for accepting key requests. - share: cross-signed-tofu - # Options for Megolm room key rotation. These options allow you to - # configure the m.room.encryption event content. See: - # https://spec.matrix.org/v1.3/client-server-api/#mroomencryption for - # more information about that event. - rotation: - # Enable custom Megolm room key rotation settings. Note that these - # settings will only apply to rooms created after this option is - # set. - enable_custom: false - # The maximum number of milliseconds a session should be used - # before changing it. The Matrix spec recommends 604800000 (a week) - # as the default. - milliseconds: 604800000 - # The maximum number of messages that should be sent with a given a - # session before changing it. The Matrix spec recommends 100 as the - # default. - messages: 100 - - # Disable rotating keys when a user's devices change? - # You should not enable this option unless you understand all the implications. - disable_device_change_key_rotation: false - - # Settings for provisioning API - provisioning: - # Prefix for the provisioning API paths. - prefix: /_matrix/provision - # Shared secret for authentication. If set to "generate", a random secret will be generated, - # or if set to "disable", the provisioning API will be disabled. - shared_secret: generate - - # Permissions for using the bridge. - # Permitted values: - # relay - Talk through the relaybot (if enabled), no access otherwise - # user - Access to use the bridge to chat with a Slack account. - # admin - User level and some additional administration tools - # Permitted keys: - # * - All Matrix users - # domain - All users on that homeserver - # mxid - Specific user permissions: "*": relay "{{ MATRIX_SERVER_NAME }}": user "@{{ MATRIX_ADMINISTRATOR_USERNAME }}:{{ MATRIX_SERVER_NAME }}": admin -# Logging config. See https://github.com/tulir/zeroconfig for details. +matrix: + message_status_events: false + delivery_receipts: true + message_error_notices: true + sync_direct_chat_list: false + federate_rooms: true + +provisioning: + shared_secret: generate + +network: +{% raw %} + displayname_template: '{{or .Profile.DisplayName .Profile.RealName .Name}}{{if .IsBot}} (bot){{end}}' + channel_name_template: '{{if and .IsChannel (not .IsPrivate)}}#{{end}}{{.Name}}{{if .IsNoteToSelf}} (you){{end}}' + team_name_template: "{{ .Name }}" +{% endraw %} + + custom_emoji_reactions: true + workspace_avatar_in_rooms: false + participant_sync_count: 5 + participant_sync_only_on_create: true + mute_channels_by_default: false + + backfill: + conversation_count: -1 + +double_puppet: + servers: + {{ MATRIX_SERVER_NAME }}: {{ MATRIX_SYNAPSE_URL }} + allow_discovery: false + secrets: + {{ MATRIX_SERVER_NAME }}: {{ MATRIX_REGISTRATION_SHARED_SECRET }} + +backfill: + enabled: false + max_initial_messages: 50 + max_catchup_messages: 500 + unread_hours_threshold: 720 + threads: + max_initial_messages: 50 + queue: + enabled: false + +encryption: + allow: false + default: false + require: false + appservice: false + allow_key_sharing: false + pickle_key: generate + verification_levels: + receive: unverified + send: unverified + share: cross-signed-tofu + rotation: + enable_custom: false + milliseconds: 604800000 + messages: 100 + disable_device_change_key_rotation: false + logging: min_level: debug writers: - - type: stdout - format: pretty-colored - - type: file - format: json - filename: ./logs/mautrix-slack.log - max_size: 100 - max_backups: 10 - compress: true \ No newline at end of file + - type: stdout + format: pretty-colored + - type: file + format: json + filename: ./logs/mautrix-slack.log + max_size: 100 + max_backups: 10 + compress: true diff --git a/roles/web-app-matrix/templates/flavor/compose/mautrix/telegram.config.yml.j2 b/roles/web-app-matrix/templates/flavor/compose/mautrix/telegram.config.yml.j2 index 4c99ffe295..2568d7e006 100644 --- a/roles/web-app-matrix/templates/flavor/compose/mautrix/telegram.config.yml.j2 +++ b/roles/web-app-matrix/templates/flavor/compose/mautrix/telegram.config.yml.j2 @@ -1,670 +1,197 @@ -# Homeserver details +bridge: + command_prefix: "!tg" + personal_filtering_spaces: true + private_chat_portal_meta: true + bridge_matrix_leave: true + bridge_notices: false + tag_only_on_create: true + kick_matrix_users: true + + relay: + enabled: false + admin_only: true + default_relays: [] + message_formats: + m.text: {% raw %}"{{ .Sender.DisambiguatedName }}: {{ .Message }}"{% endraw %} + + m.notice: {% raw %}"{{ .Sender.DisambiguatedName }}: {{ .Message }}"{% endraw %} + + m.emote: {% raw %}"* {{ .Sender.DisambiguatedName }} {{ .Message }}"{% endraw %} + + m.file: {% raw %}"{{ .Sender.DisambiguatedName }} sent a file{{ if .Caption }}: {{ .Caption }}{{ end }}"{% endraw %} + + m.image: {% raw %}"{{ .Sender.DisambiguatedName }} sent an image{{ if .Caption }}: {{ .Caption }}{{ end }}"{% endraw %} + + m.audio: {% raw %}"{{ .Sender.DisambiguatedName }} sent an audio file{{ if .Caption }}: {{ .Caption }}{{ end }}"{% endraw %} + + m.video: {% raw %}"{{ .Sender.DisambiguatedName }} sent a video{{ if .Caption }}: {{ .Caption }}{{ end }}"{% endraw %} + + m.location: {% raw %}"{{ .Sender.DisambiguatedName }} sent a location{{ if .Caption }}: {{ .Caption }}{{ end }}"{% endraw %} + + portal_create_filter: + mode: deny + list: [] + always_deny_from_login: [] + + management_room_text: + welcome: "Hello, I'm a Telegram bridge bot." + welcome_connected: "Use `help` for help." + welcome_unconnected: "Use `help` for help or `login` to log in." + additional_help: "" + + permissions: + "*": "relay" + "public.{{ MATRIX_SERVER_NAME }}": "commands" + "{{ MATRIX_SERVER_NAME }}": "user" + "@{{ MATRIX_ADMINISTRATOR_USERNAME }}:{{ MATRIX_SERVER_NAME }}": "admin" + +database: + type: postgres + uri: postgres://mautrix_telegram_bridge:{{ lookup('config', application_id, 'credentials.mautrix_telegram_bridge_database_password') | regex_replace('/', '%2F') | regex_replace('@', '%40') | regex_replace(':', '%3A') }}@{{ MATRIX_POSTGRES_HOST }}/mautrix_telegram_bridge?sslmode=disable + max_open_conns: 5 + max_idle_conns: 1 + max_conn_idle_time: null + max_conn_lifetime: null + homeserver: - # The address that this appservice can use to connect to the homeserver. address: http://synapse:{{ MATRIX_SYNAPSE_CONTAINER_PORT }} - # The domain of the homeserver (for MXIDs, etc). domain: {{ MATRIX_SERVER_NAME }} - # Whether or not to verify the SSL certificate of the homeserver. - # Only applies if address starts with https:// - verify_ssl: true - # What software is the homeserver running? - # Standard Matrix homeservers like Synapse, Dendrite and Conduit should just use "standard" here. software: standard - # Number of retries for all HTTP requests if the homeserver isn't reachable. - http_retry_count: 4 - # The URL to push real-time bridge status to. - # If set, the bridge will make POST requests to this URL whenever a user's Telegram connection state changes. - # The bridge will use the appservice as_token to authorize requests. - status_endpoint: null - # Endpoint for reporting per-message status. - message_send_checkpoint_endpoint: null - # Whether asynchronous uploads via MSC2246 should be enabled for media. - # Requires a media repo that supports MSC2246. + status_endpoint: + message_send_checkpoint_endpoint: async_media: false + retry_limit: 4 -# Application service host/registration related details -# Changing these values requires regeneration of the registration. appservice: - # The address that the homeserver can use to connect to this appservice. address: http://mautrix-telegram:29317 - # When using https:// the TLS certificate and key files for the address. - tls_cert: false - tls_key: false - - # The hostname and port where this appservice should listen. hostname: 0.0.0.0 port: 29317 - # The maximum body size of appservice API requests (from the homeserver) in mebibytes - # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s - max_body_size: 1 - # The full URI to the database. SQLite and Postgres are supported. - # Format examples: - # SQLite: sqlite:filename.db - # Postgres: postgres://username:password@hostname/dbname - database: postgres://mautrix_telegram_bridge:{{ lookup('config', application_id, 'credentials.mautrix_telegram_bridge_database_password')}}@{{ MATRIX_POSTGRES_HOST }}/mautrix_telegram_bridge - # Additional arguments for asyncpg.create_pool() or sqlite3.connect() - # https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool - # https://docs.python.org/3/library/sqlite3.html#sqlite3.connect - # For sqlite, min_size is used as the connection thread pool size and max_size is ignored. - # Additionally, SQLite supports init_commands as an array of SQL queries to run on connect (e.g. to set PRAGMAs). - database_opts: - min_size: 1 - max_size: 10 - - # Public part of web server for out-of-Matrix interaction with the bridge. - # Used for things like login if the user wants to make sure the 2FA password isn't stored in - # the HS database. - public: - # Whether or not the public-facing endpoints should be enabled. - enabled: false - # The prefix to use in the public-facing endpoints. - prefix: /public - # The base URL where the public-facing endpoints are available. The prefix is not added - # implicitly. - external: {{ lookup('tls', application_id, 'protocols.web') }}://{{ MATRIX_SERVER_NAME }}/public - - # Provisioning API part of the web server for automated portal creation and fetching information. - # Used by things like mautrix-manager (https://github.com/tulir/mautrix-manager). - provisioning: - # Whether or not the provisioning API should be enabled. - enabled: true - # The prefix to use in the provisioning API endpoints. - prefix: /_matrix/provision - # The shared secret to authorize users of the API. - # Set to "generate" to generate and save a new token. - shared_secret: generate - - # The unique ID of this appservice. id: telegram - # Username of the appservice bot. - bot_username: telegrambot - # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty - # to leave display name/avatar as-is. - bot_displayname: Telegram bridge bot - bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX + bot: + username: telegrambot + displayname: Telegram bridge bot + avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX - # Whether or not to receive ephemeral events via appservice transactions. - # Requires MSC2409 support (i.e. Synapse 1.22+). - # You should disable bridge -> sync_with_custom_puppets when this is enabled. ephemeral_events: true - # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify. as_token: "This value is generated when generating the registration" hs_token: "This value is generated when generating the registration" -# Prometheus telemetry config. Requires prometheus-client to be installed. -metrics: - enabled: false - listen_port: 8000 + username_template: {% raw %}telegram_{{.}}{% endraw %} -# Manhole config. -manhole: - # Whether or not opening the manhole is allowed. - enabled: false - # The path for the unix socket. - path: /var/tmp/mautrix-telegram.manhole - # The list of UIDs who can be added to the whitelist. - # If empty, any UIDs can be specified in the open-manhole command. - whitelist: - - 0 - -# Bridge config -bridge: - # Localpart template of MXIDs for Telegram users. - # {userid} is replaced with the user ID of the Telegram user. - username_template: "telegram_{userid}" - # Localpart template of room aliases for Telegram portal rooms. - # {groupname} is replaced with the name part of the public channel/group invite link ( https://t.me/{} ) - alias_template: "telegram_{groupname}" - # Displayname template for Telegram users. - # {displayname} is replaced with the display name of the Telegram user. - displayname_template: "{displayname} (Telegram)" - - # Set the preferred order of user identifiers which to use in the Matrix puppet display name. - # In the (hopefully unlikely) scenario that none of the given keys are found, the numeric user - # ID is used. - # - # If the bridge is working properly, a phone number or an username should always be known, but - # the other one can very well be empty. - # - # Valid keys: - # "full name" (First and/or last name) - # "full name reversed" (Last and/or first name) - # "first name" - # "last name" - # "username" - # "phone number" - displayname_preference: - - full name - - username - - phone number - # Maximum length of displayname - displayname_max_length: 100 - # Remove avatars from Telegram ghost users when removed on Telegram. This is disabled by default - # as there's no way to determine whether an avatar is removed or just hidden from some users. If - # you're on a single-user instance, this should be safe to enable. - allow_avatar_remove: false - # Should contact names and profile pictures be allowed? - # This is only safe to enable on single-user instances. - allow_contact_info: false - - # Maximum number of members to sync per portal when starting up. Other members will be - # synced when they send messages. The maximum is 10000, after which the Telegram server - # will not send any more members. - # -1 means no limit (which means it's limited to 10000 by the server) - max_initial_member_sync: 100 - # Maximum number of participants in chats to bridge. Only applies when the portal is being created. - # If there are more members when trying to create a room, the room creation will be cancelled. - # -1 means no limit (which means all chats can be bridged) - max_member_count: -1 - # Whether or not to sync the member list in channels. - # If no channel admins have logged into the bridge, the bridge won't be able to sync the member - # list regardless of this setting. - sync_channel_members: false - # Whether or not to skip deleted members when syncing members. - skip_deleted_members: true - # Whether or not to automatically synchronize contacts and chats of Matrix users logged into - # their Telegram account at startup. - startup_sync: false - # Number of most recently active dialogs to check when syncing chats. - # Set to 0 to remove limit. - sync_update_limit: 0 - # Number of most recently active dialogs to create portals for when syncing chats. - # Set to 0 to remove limit. - sync_create_limit: 15 - # Should all chats be scheduled to be created later? - # This is best used in combination with MSC2716 infinite backfill. - sync_deferred_create_all: false - # Whether or not to sync and create portals for direct chats at startup. - sync_direct_chats: false - # The maximum number of simultaneous Telegram deletions to handle. - # A large number of simultaneous redactions could put strain on your homeserver. - max_telegram_delete: 10 - # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames) - # at startup and when creating a bridge. - sync_matrix_state: true - # Allow logging in within Matrix. If false, users can only log in using login-qr or the - # out-of-Matrix login website (see appservice.public config section) - allow_matrix_login: true - # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix. - public_portals: false - # Whether or not to use /sync to get presence, read receipts and typing notifications - # when double puppeting is enabled - sync_with_custom_puppets: false - # Whether or not to update the m.direct account data event when double puppeting is enabled. - # Note that updating the m.direct event is not atomic (except with mautrix-asmux) - # and is therefore prone to race conditions. - sync_direct_chat_list: false - # Servers to always allow double puppeting from - double_puppet_server_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_SYNAPSE_URL }} - # Allow using double puppeting from any server with a valid client .well-known file. - double_puppet_allow_discovery: false - # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth - # - # If set, custom puppets will be enabled automatically for local users - # instead of users having to find an access token and run `login-matrix` - # manually. - # If using this for other servers than the bridge's server, - # you must also set the URL in the double_puppet_server_map. - login_shared_secret_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_REGISTRATION_SHARED_SECRET }} - # Set to false to disable link previews in messages sent to Telegram. - telegram_link_preview: true - # Whether or not the !tg join command should do a HTTP request - # to resolve redirects in invite links. - invite_link_resolve: false - # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552. - # This is currently not supported in most clients. - caption_in_message: false - # Maximum size of image in megabytes before sending to Telegram as a document. - image_as_file_size: 10 - # Maximum number of pixels in an image before sending to Telegram as a document. Defaults to 4096x4096 = 16777216. - image_as_file_pixels: 16777216 - # Enable experimental parallel file transfer, which makes uploads/downloads much faster by - # streaming from/to Matrix and using many connections for Telegram. - # Note that generating HQ thumbnails for videos is not possible with streamed transfers. - # This option uses internal Telethon implementation details and may break with minor updates. - parallel_file_transfer: false - # Whether or not created rooms should have federation enabled. - # If false, created portal rooms will never be federated. - federate_rooms: true - # Should the bridge send all unicode reactions as custom emoji reactions to Telegram? - # By default, the bridge only uses custom emojis for unicode emojis that aren't allowed in reactions. - always_custom_emoji_reaction: false - # Settings for converting animated stickers. - animated_sticker: - # Format to which animated stickers should be converted. - # disable - No conversion, send as-is (gzipped lottie) - # png - converts to non-animated png (fastest), - # gif - converts to animated gif - # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support - # webp - converts to animated webp, requires ffmpeg executable with webp codec/container support - target: gif - # Should video stickers be converted to the specified format as well? - convert_from_webm: false - # Arguments for converter. All converters take width and height. - args: - width: 256 - height: 256 - fps: 25 # only for webm, webp and gif (2, 5, 10, 20 or 25 recommended) - # Settings for converting animated emoji. - # Same as animated_sticker, but webm is not supported as the target - # (because inline images can only contain images, not videos). - animated_emoji: - target: webp - args: - width: 64 - height: 64 - fps: 25 - # End-to-bridge encryption support options. - # - # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info. - encryption: - # Allow encryption, work in group chat rooms with e2ee enabled - allow: false - # Default to encryption, force-enable encryption in all portals the bridge creates - # This will cause the bridge bot to be in private chats for the encryption to work properly. - default: false - # Whether to use MSC2409/MSC3202 instead of /sync long polling for receiving encryption-related data. - appservice: false - # Require encryption, drop any unencrypted messages. - require: false - # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled. - # You must use a client that supports requesting keys from other users to use this feature. - allow_key_sharing: false - # Options for deleting megolm sessions from the bridge. - delete_keys: - # Beeper-specific: delete outbound sessions when hungryserv confirms - # that the user has uploaded the key to key backup. - delete_outbound_on_ack: false - # Don't store outbound sessions in the inbound table. - dont_store_outbound: false - # Ratchet megolm sessions forward after decrypting messages. - ratchet_on_decrypt: false - # Delete fully used keys (index >= max_messages) after decrypting messages. - delete_fully_used_on_decrypt: false - # Delete previous megolm sessions from same device when receiving a new one. - delete_prev_on_new_session: false - # Delete megolm sessions received from a device when the device is deleted. - delete_on_device_delete: false - # Periodically delete megolm sessions when 2x max_age has passed since receiving the session. - periodically_delete_expired: false - # Delete inbound megolm sessions that don't have the received_at field used for - # automatic ratcheting and expired session deletion. This is meant as a migration - # to delete old keys prior to the bridge update. - delete_outdated_inbound: false - # What level of device verification should be required from users? - # - # Valid levels: - # unverified - Send keys to all device in the room. - # cross-signed-untrusted - Require valid cross-signing, but trust all cross-signing keys. - # cross-signed-tofu - Require valid cross-signing, trust cross-signing keys on first use (and reject changes). - # cross-signed-verified - Require valid cross-signing, plus a valid user signature from the bridge bot. - # Note that creating user signatures from the bridge bot is not currently possible. - # verified - Require manual per-device verification - # (currently only possible by modifying the `trust` column in the `crypto_device` database table). - verification_levels: - # Minimum level for which the bridge should send keys to when bridging messages from Telegram to Matrix. - receive: unverified - # Minimum level that the bridge should accept for incoming Matrix messages. - send: unverified - # Minimum level that the bridge should require for accepting key requests. - share: cross-signed-tofu - # Options for Megolm room key rotation. These options allow you to - # configure the m.room.encryption event content. See: - # https://spec.matrix.org/v1.3/client-server-api/#mroomencryption for - # more information about that event. - rotation: - # Enable custom Megolm room key rotation settings. Note that these - # settings will only apply to rooms created after this option is - # set. - enable_custom: false - # The maximum number of milliseconds a session should be used - # before changing it. The Matrix spec recommends 604800000 (a week) - # as the default. - milliseconds: 604800000 - # The maximum number of messages that should be sent with a given a - # session before changing it. The Matrix spec recommends 100 as the - # default. - messages: 100 - - # Disable rotating keys when a user's devices change? - # You should not enable this option unless you understand all the implications. - disable_device_change_key_rotation: false - - # Whether to explicitly set the avatar and room name for private chat portal rooms. - # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms. - # If set to `always`, all DM rooms will have explicit names and avatars set. - # If set to `never`, DM rooms will never have names and avatars set. - private_chat_portal_meta: default - # Disable generating reply fallbacks? Some extremely bad clients still rely on them, - # but they're being phased out and will be completely removed in the future. - disable_reply_fallbacks: false - # Should cross-chat replies from Telegram be bridged? Most servers and clients don't support this. - cross_room_replies: false - # Whether or not the bridge should send a read receipt from the bridge bot when a message has - # been sent to Telegram. - delivery_receipts: false - # Whether or not delivery errors should be reported as messages in the Matrix room. - delivery_error_reports: false - # Should errors in incoming message handling send a message to the Matrix room? - incoming_bridge_error_reports: false - # Whether the bridge should send the message status as a custom com.beeper.message_send_status event. +matrix: message_status_events: false - # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run. - # This field will automatically be changed back to false after it, - # except if the config file is not writable. - resend_bridge_info: false - # When using double puppeting, should muted chats be muted in Matrix? - mute_bridging: false - # When using double puppeting, should pinned chats be moved to a specific tag in Matrix? - # The favorites tag is `m.favourite`. - pinned_tag: null - # Same as above for archived chats, the low priority tag is `m.lowpriority`. - archive_tag: null - # Whether or not mute status and tags should only be bridged when the portal room is created. - tag_only_on_create: true - # Should leaving the room on Matrix make the user leave on Telegram? - bridge_matrix_leave: true - # Should the user be kicked out of all portals when logging out of the bridge? - kick_on_logout: true - # Should the "* user joined Telegram" notice always be marked as read automatically? - always_read_joined_telegram_notice: true - # Should the bridge auto-create a group chat on Telegram when a ghost is invited to a room? - # Requires the user to have sufficient power level and double puppeting enabled. - create_group_on_invite: true - # Settings for backfilling messages from Telegram. - backfill: - # Allow backfilling at all? - enable: true - # Whether or not to enable backfilling in normal groups. - # Normal groups have numerous technical problems in Telegram, and backfilling normal groups - # will likely cause problems if there are multiple Matrix users in the group. - normal_groups: false + delivery_receipts: false + message_error_notices: true + sync_direct_chat_list: false + federate_rooms: true - # If a backfilled chat is older than this number of hours, mark it as read even if it's unread on Telegram. - # Set to -1 to let any chat be unread. - unread_hours_threshold: 720 +provisioning: + shared_secret: generate + allow_matrix_auth: true - # Forward backfilling limits. - # - # Using a negative initial limit is not recommended, as it would try to backfill everything in a single batch. - forward_limits: - # Number of messages to backfill immediately after creating a portal. - initial: - user: 50 - normal_group: 100 - supergroup: 10 - channel: 10 - # Number of messages to backfill when syncing chats. - sync: - user: 100 - normal_group: 100 - supergroup: 100 - channel: 100 - # Timeout for forward backfills in seconds. If you have a high limit, you'll have to increase this too. - forward_timeout: 900 +backfill: + enabled: true + max_initial_messages: 50 + max_catchup_messages: 500 + unread_hours_threshold: 720 + threads: + max_initial_messages: 50 + queue: + enabled: false + batch_size: 100 + batch_delay: 20 + max_batches: -1 - # Settings for incremental backfill of history. These only apply to Beeper, as upstream abandoned MSC2716. - incremental: - # Maximum number of messages to backfill per batch. - messages_per_batch: 100 - # The number of seconds to wait after backfilling the batch of messages. - post_batch_delay: 20 - # The maximum number of batches to backfill per portal, split by the chat type. - # If set to -1, all messages in the chat will eventually be backfilled. - max_batches: - # Direct chats - user: -1 - # Normal groups. Note that the normal_groups option above must be enabled - # for these to be backfilled. - normal_group: -1 - # Supergroups - supergroup: 10 - # Broadcast channels - channel: -1 +double_puppet: + servers: + {{ MATRIX_SERVER_NAME }}: {{ MATRIX_SYNAPSE_URL }} + allow_discovery: false + secrets: + {{ MATRIX_SERVER_NAME }}: {{ MATRIX_REGISTRATION_SHARED_SECRET }} - # Overrides for base power levels. - initial_power_level_overrides: - user: {} - group: {} +encryption: + allow: false + default: false + require: false + appservice: false + allow_key_sharing: false + pickle_key: generate + verification_levels: + receive: unverified + send: unverified + share: cross-signed-tofu + rotation: + enable_custom: false + milliseconds: 604800000 + messages: 100 + disable_device_change_key_rotation: false - # Whether to bridge Telegram bot messages as m.notices or m.texts. - bot_messages_as_notices: true - bridge_notices: - # Whether or not Matrix bot messages (type m.notice) should be bridged. - default: false - # List of user IDs for whom the previous flag is flipped. - # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but - # notices from users listed here will be bridged. - exceptions: [] +network: + api_id: {{ lookup('config', application_id, 'credentials.mautrix_telgegram_bridge_api_id') }} + api_hash: {{ lookup('config', application_id, 'credentials.mautrix_telgegram_bridge_api_pin') }} - # An array of possible values for the $distinguisher variable in message formats. - # Each user gets one of the values here, based on a hash of their user ID. - # If the array is empty, the $distinguisher variable will also be empty. - relay_user_distinguishers: ["๐ŸŸฆ", "๐ŸŸฃ", "๐ŸŸฉ", "โญ•๏ธ", "๐Ÿ”ถ", "โฌ›๏ธ", "๐Ÿ”ต", "๐ŸŸข"] - # The formats to use when sending messages to Telegram via the relay bot. - # Text msgtypes (m.text, m.notice and m.emote) support HTML, media msgtypes don't. - # - # Available variables: - # $sender_displayname - The display name of the sender (e.g. Example User) - # $sender_username - The username (Matrix ID localpart) of the sender (e.g. exampleuser) - # $sender_mxid - The Matrix ID of the sender (e.g. @exampleuser:example.com) - # $distinguisher - A random string from the options in the relay_user_distinguishers array. - # $message - The message content - message_formats: - m.text: "$distinguisher $sender_displayname: $message" - m.notice: "$distinguisher $sender_displayname: $message" - m.emote: "* $distinguisher $sender_displayname $message" - m.file: "$distinguisher $sender_displayname sent a file: $message" - m.image: "$distinguisher $sender_displayname sent an image: $message" - m.audio: "$distinguisher $sender_displayname sent an audio file: $message" - m.video: "$distinguisher $sender_displayname sent a video: $message" - m.location: "$distinguisher $sender_displayname sent a location: $message" - # Telegram doesn't have built-in emotes, this field specifies how m.emote's from authenticated - # users are sent to telegram. All fields in message_formats are supported. Additionally, the - # Telegram user info is available in the following variables: - # $displayname - Telegram displayname - # $username - Telegram username (may not exist) - # $mention - Telegram @username or displayname mention (depending on which exists) - emote_format: "* $mention $formatted_body" - - # The formats to use when sending state events to Telegram via the relay bot. - # - # Variables from `message_formats` that have the `sender_` prefix are available without the prefix. - # In name_change events, `$prev_displayname` is the previous displayname. - # - # Set format to an empty string to disable the messages for that event. - state_event_formats: - join: "$distinguisher $displayname joined the room." - leave: "$distinguisher $displayname left the room." - name_change: "$distinguisher $prev_displayname changed their name to $distinguisher $displayname" - - # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and - # `filter-mode` management commands. - # - # An empty blacklist will essentially disable the filter. - filter: - # Filter mode to use. Either "blacklist" or "whitelist". - # If the mode is "blacklist", the listed chats will never be bridged. - # If the mode is "whitelist", only the listed chats can be bridged. - mode: blacklist - # The list of group/channel IDs to filter. - list: [] - # How to handle direct chats: - # If users is "null", direct chats will follow the previous settings. - # If users is "true", direct chats will always be bridged. - # If users is "false", direct chats will never be bridged. - users: true - - # The prefix for commands. Only required in non-management rooms. - command_prefix: "!tg" - - # Messages sent upon joining a management room. - # Markdown is supported. The defaults are listed below. - management_room_text: - # Sent when joining a room. - welcome: "Hello, I'm a Telegram bridge bot." - # Sent when joining a management room and the user is already logged in. - welcome_connected: "Use `help` for help." - # Sent when joining a management room and the user is not logged in. - welcome_unconnected: "Use `help` for help or `login` to log in." - # Optional extra text sent when joining a management room. - additional_help: "" - - # Send each message separately (for readability in some clients) - management_room_multiple_messages: false - - # Permissions for using the bridge. - # Permitted values: - # relaybot - Only use the bridge via the relaybot, no access to commands. - # user - Relaybot level + access to commands to create bridges. - # puppeting - User level + logging in with a Telegram account. - # full - Full access to use the bridge, i.e. previous levels + Matrix login. - # admin - Full access to use the bridge and some extra administration commands. - # Permitted keys: - # * - All Matrix users - # domain - All users on that homeserver - # mxid - Specific user - permissions: - "*": "relaybot" - "public.{{ MATRIX_SERVER_NAME }}": "user" - "{{ MATRIX_SERVER_NAME }}": "full" - "@{{ MATRIX_ADMINISTRATOR_USERNAME }}:{{ MATRIX_SERVER_NAME }}": "admin" - - # Options related to the message relay Telegram bot. - relaybot: - private_chat: - # List of users to invite to the portal when someone starts a private chat with the bot. - # If empty, private chats with the bot won't create a portal. - invite: [] - # Whether or not to bridge state change messages in relaybot private chats. - state_changes: true - # When private_chat_invite is empty, this message is sent to users /starting the - # relaybot. Telegram's "markdown" is supported. - message: This is a Matrix bridge relaybot and does not support direct chats - # List of users to invite to all group chat portals created by the bridge. - group_chat_invite: [] - # Whether or not the relaybot should not bridge events in unbridged group chats. - # If false, portals will be created when the relaybot receives messages, just like normal - # users. This behavior is usually not desirable, as it interferes with manually bridging - # the chat to another room. - ignore_unbridged_group_chat: true - # Whether or not to allow creating portals from Telegram. - authless_portals: true - # Whether or not to allow Telegram group admins to use the bot commands. - whitelist_group_admins: true - # Whether or not to ignore incoming events sent by the relay bot. - ignore_own_incoming_events: true - # List of usernames/user IDs who are also allowed to use the bot commands. - whitelist: [] - -# Telegram config -telegram: - # Get your own API keys at https://my.telegram.org/apps - api_id: {{ lookup('config', application_id, 'credentials.mautrix_telgegram_bridge_api_id')}} - api_hash: {{ lookup('config', application_id, 'credentials.mautrix_telgegram_bridge_api_pin')}} - # (Optional) Create your own bot at https://t.me/BotFather - bot_token: disabled - - # Should the bridge request missed updates from Telegram when restarting? - catch_up: true - # Should incoming updates be handled sequentially to make sure order is preserved on Matrix? - sequential_updates: true - exit_on_update_error: false - - # Telethon connection options. - connection: - # The timeout in seconds to be used when connecting. - timeout: 120 - # How many times the reconnection should retry, either on the initial connection or when - # Telegram disconnects us. May be set to a negative or null value for infinite retries, but - # this is not recommended, since the program can get stuck in an infinite loop. - retries: 5 - # The delay in seconds to sleep between automatic reconnections. - retry_delay: 1 - # The threshold below which the library should automatically sleep on flood wait errors - # (inclusive). For instance, if a FloodWaitError for 17s occurs and flood_sleep_threshold - # is 20s, the library will sleep automatically. If the error was for 21s, it would raise - # the error instead. Values larger than a day (86400) will be changed to a day. - flood_sleep_threshold: 60 - # How many times a request should be retried. Request are retried when Telegram is having - # internal issues, when there is a FloodWaitError less than flood_sleep_threshold, or when - # there's a migrate error. May take a negative or null value for infinite retries, but this - # is not recommended, since some requests can always trigger a call fail (such as searching - # for messages). - request_retries: 5 - # Use IPv6 for Telethon connection - use_ipv6: {{ NETWORK_IPV6_ENABLED | bool }} - - # Device info sent to Telegram. device_info: - # "auto" = OS name+version. device_model: mautrix-telegram - # "auto" = Telethon version. - system_version: auto - # "auto" = mautrix-telegram version. + system_version: app_version: auto lang_code: en system_lang_code: en - # Custom server to connect to. - server: - # Set to true to use these server settings. If false, will automatically - # use production server assigned by Telegram. Set to false in production. - enabled: false - # The DC ID to connect to. - dc: 2 - # The IP to connect to. - ip: 149.154.167.40 - # The port to connect to. 443 may not work, 80 is better and both are equally secure. - port: 80 + animated_sticker: + target: gif + convert_from_webm: false + args: + width: 256 + height: 256 + fps: 25 + + member_list: + max_initial_sync: 100 + sync_broadcast_channels: false + skip_deleted: true + + ping: + interval_seconds: 30 + timeout_seconds: 10 - # Telethon proxy configuration. - # You must install PySocks from pip for proxies to work. proxy: - # Allowed types: disabled, socks4, socks5, http, mtproxy type: disabled - # Proxy IP address and port. - address: 127.0.0.1 - port: 1080 - # Whether or not to perform DNS resolving remotely. Only for socks/http proxies. - rdns: true - # Proxy authentication (optional). Put MTProxy secret in password field. - username: "" - password: "" + address: "127.0.0.1:1080" + username: + password: + + sync: + update_limit: 100 + create_limit: 15 + login_sync_limit: 15 + direct_chats: true + + takeout: + dialog_sync: false + forward_backfill: false + backward_backfill: false + + max_member_count: -1 + contact_avatars: false + contact_names: false + always_custom_emoji_reaction: false + saved_message_avatar: mxc://maunium.net/XhhfHoPejeneOngMyBbtyWDk + always_tombstone_on_supergroup_migration: false + image_as_file_pixels: 16777216 + disable_view_once: false + video_url_preview_as_file: false + displayname_template: {% raw %}"{{ if .Deleted }}Deleted account {{ .UserID }}{{ else }}{{ .FullName }}{{ end }}"{% endraw %} -# Python logging configuration. -# -# See section 16.7.2 of the Python documentation for more info: -# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema logging: - version: 1 - formatters: - colored: - (): mautrix_telegram.util.ColorFormatter - format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" - normal: - format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" - handlers: - file: - class: logging.handlers.RotatingFileHandler - formatter: normal - filename: ./mautrix-telegram.log - maxBytes: 10485760 - backupCount: 10 - console: - class: logging.StreamHandler - formatter: colored - loggers: - mau: - level: DEBUG - telethon: - level: INFO - aiohttp: - level: INFO - root: - level: DEBUG - handlers: [file, console] \ No newline at end of file + min_level: debug + writers: + - type: stdout + format: pretty-colored + - type: file + format: json + filename: ./logs/bridge.log + max_size: 100 + max_backups: 10 + compress: false diff --git a/roles/web-app-matrix/templates/flavor/compose/mautrix/whatsapp.config.yml.j2 b/roles/web-app-matrix/templates/flavor/compose/mautrix/whatsapp.config.yml.j2 index 17944f0c50..15d35bb60e 100644 --- a/roles/web-app-matrix/templates/flavor/compose/mautrix/whatsapp.config.yml.j2 +++ b/roles/web-app-matrix/templates/flavor/compose/mautrix/whatsapp.config.yml.j2 @@ -1,465 +1,256 @@ -# Homeserver details. -homeserver: - # The address that this appservice can use to connect to the homeserver. - address: http://synapse:{{ MATRIX_SYNAPSE_CONTAINER_PORT }} - # The domain of the homeserver (also known as server_name, used for MXIDs, etc). - domain: {{ MATRIX_SERVER_NAME }} - - # What software is the homeserver running? - # Standard Matrix homeservers like Synapse, Dendrite and Conduit should just use "standard" here. - software: standard - # The URL to push real-time bridge status to. - # If set, the bridge will make POST requests to this URL whenever a user's whatsapp connection state changes. - # The bridge will use the appservice as_token to authorize requests. - status_endpoint: null - # Endpoint for reporting per-message status. - message_send_checkpoint_endpoint: null - # Does the homeserver support https://github.com/matrix-org/matrix-spec-proposals/pull/2246? - async_media: false - # Should the bridge use a websocket for connecting to the homeserver? - # The server side is currently not documented anywhere and is only implemented by mautrix-wsproxy, - # mautrix-asmux (deprecated), and hungryserv (proprietary). - websocket: false - # How often should the websocket be pinged? Pinging will be disabled if this is zero. - ping_interval_seconds: 0 -# Application service host/registration related details. -# Changing these values requires regeneration of the registration. -appservice: - # The address that the homeserver can use to connect to this appservice. - address: http://mautrix-whatsapp:29318 - - # The hostname and port where this appservice should listen. - hostname: 0.0.0.0 - port: 29318 - - # Database config. - database: - # The database type. "sqlite3-fk-wal" and "postgres" are supported. - type: postgres - # The database URI. - # SQLite: A raw file path is supported, but `file:?_txlock=immediate` is recommended. - # https://github.com/mattn/go-sqlite3#connection-string - # Postgres: Connection string. For example, postgres://user:password@host/database?sslmode=disable - # To connect via Unix socket, use something like postgres:///dbname?host=/var/run/postgresql - uri: postgres://mautrix_whatsapp_bridge:{{ lookup('config', application_id, 'credentials.mautrix_whatsapp_bridge_database_password')}}@{{ MATRIX_POSTGRES_HOST }}/mautrix_whatsapp_bridge?sslmode=disable - # Maximum number of connections. Mostly relevant for Postgres. - max_open_conns: 20 - max_idle_conns: 2 - # Maximum connection idle time and lifetime before they're closed. Disabled if null. - # Parsed with https://pkg.go.dev/time#ParseDuration - max_conn_idle_time: null - max_conn_lifetime: null - - # The unique ID of this appservice. - id: whatsapp - # Appservice bot details. - bot: - # Username of the appservice bot. - username: whatsappbot - # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty - # to leave display name/avatar as-is. - displayname: WhatsApp bridge bot - avatar: mxc://maunium.net/NeXNQarUbrlYBiPCpprYsRqr - # Whether or not to receive ephemeral events via appservice transactions. - # Requires MSC2409 support (i.e. Synapse 1.22+). - ephemeral_events: true - # Should incoming events be handled asynchronously? - # This may be necessary for large public instances with lots of messages going through. - # However, messages will not be guaranteed to be bridged in the same order they were sent in. - async_transactions: false - - # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify. - as_token: "This value is generated when generating the registration" - hs_token: "This value is generated when generating the registration" - -# Segment-compatible analytics endpoint for tracking some events, like provisioning API login and encryption errors. -analytics: - # Hostname of the tracking server. The path is hardcoded to /v1/track - host: api.segment.io - # API key to send with tracking requests. Tracking is disabled if this is null. - token: null - # Optional user ID for tracking events. If null, defaults to using Matrix user ID. - user_id: null - -# Prometheus config. -metrics: - # Enable prometheus metrics? - enabled: false - # IP and port where the metrics listener should be. The path is always /metrics - listen: 127.0.0.1:8001 - -# Config for things that are directly sent to WhatsApp. -whatsapp: - # Device name that's shown in the "WhatsApp Web" section in the mobile app. +network: os_name: Mautrix-WhatsApp bridge - # Browser name that determines the logo shown in the mobile app. - # Must be "unknown" for a generic icon or a valid browser name if you want a specific icon. - # List of valid browser names: https://github.com/tulir/whatsmeow/blob/efc632c008604016ddde63bfcfca8de4e5304da9/binary/proto/def.proto#L43-L64 browser_name: unknown -# Bridge config -bridge: -{% raw %} - # Localpart template of MXIDs for WhatsApp users. - # {{.}} is replaced with the phone number of the WhatsApp user. - username_template: whatsapp_{{.}} - # Displayname template for WhatsApp users. - # {{.PushName}} - nickname set by the WhatsApp user - # {{.BusinessName}} - validated WhatsApp business name - # {{.Phone}} - phone number (international format) - # The following variables are also available, but will cause problems on multi-user instances: - # {{.FullName}} - full name from contact list - # {{.FirstName}} - first name from contact list - displayname_template: "{{or .BusinessName .PushName .JID}} (WA)" -{% endraw %} - # Should the bridge create a space for each logged-in user and add bridged rooms to it? - # Users who logged in before turning this on should run `!wa sync space` to create and fill the space for the first time. - personal_filtering_spaces: false - # Should the bridge send a read receipt from the bridge bot when a message has been sent to WhatsApp? - delivery_receipts: false - # Whether the bridge should send the message status as a custom com.beeper.message_send_status event. - message_status_events: false - # Whether the bridge should send error notices via m.notice events when a message fails to bridge. - message_error_notices: true - # Should incoming calls send a message to the Matrix room? - call_start_notices: true - # Should another user's cryptographic identity changing send a message to Matrix? - identity_change_notices: false - portal_message_buffer: 128 - # Settings for handling history sync payloads. - history_sync: - # Enable backfilling history sync payloads from WhatsApp? - backfill: true - # The maximum number of initial conversations that should be synced. - # Other conversations will be backfilled on demand when receiving a message or when initiating a direct chat. - max_initial_conversations: -1 - # Maximum number of messages to backfill in each conversation. - # Set to -1 to disable limit. - message_count: 50 - # Should the bridge request a full sync from the phone when logging in? - # This bumps the size of history syncs from 3 months to 1 year. - request_full_sync: false - # Configuration parameters that are sent to the phone along with the request full sync flag. - # By default (when the values are null or 0), the config isn't sent at all. - full_sync_config: - # Number of days of history to request. - # The limit seems to be around 3 years, but using higher values doesn't break. - days_limit: null - # This is presumably the maximum size of the transferred history sync blob, which may affect what the phone includes in the blob. - size_mb_limit: null - # This is presumably the local storage quota, which may affect what the phone includes in the history sync blob. - storage_quota_mb: null - # If this value is greater than 0, then if the conversation's last message was more than - # this number of hours ago, then the conversation will automatically be marked it as read. - # Conversations that have a last message that is less than this number of hours ago will - # have their unread status synced from WhatsApp. - unread_hours_threshold: 0 - ############################################################################### - # The settings below are only applicable for backfilling using batch sending, # - # which is no longer supported in Synapse. # - ############################################################################### + proxy: null + get_proxy_url: null + proxy_only_login: false - # Settings for media requests. If the media expired, then it will not be on the WA servers. - # Media can always be requested by reacting with the โ™ป๏ธ (recycle) emoji. - # These settings determine if the media requests should be done automatically during or after backfill. - media_requests: - # Should expired media be automatically requested from the server as part of the backfill process? - auto_request_media: true - # Whether to request the media immediately after the media message is backfilled ("immediate") - # or at a specific time of the day ("local_time"). - request_method: immediate - # If request_method is "local_time", what time should the requests be sent (in minutes after midnight)? - request_local_time: 120 - # Settings for immediate backfills. These backfills should generally be small and their main purpose is - # to populate each of the initial chats (as configured by max_initial_conversations) with a few messages - # so that you can continue conversations without losing context. - immediate: - # The number of concurrent backfill workers to create for immediate backfills. - # Note that using more than one worker could cause the room list to jump around - # since there are no guarantees about the order in which the backfills will complete. - worker_count: 1 - # The maximum number of events to backfill initially. - max_events: 10 - # Settings for deferred backfills. The purpose of these backfills are to fill in the rest of - # the chat history that was not covered by the immediate backfills. - # These backfills generally should happen at a slower pace so as not to overload the homeserver. - # Each deferred backfill config should define a "stage" of backfill (i.e. the last week of messages). - # The fields are as follows: - # - start_days_ago: the number of days ago to start backfilling from. - # To indicate the start of time, use -1. For example, for a week ago, use 7. - # - max_batch_events: the number of events to send per batch. - # - batch_delay: the number of seconds to wait before backfilling each batch. - deferred: - # Last Week - - start_days_ago: 7 - max_batch_events: 20 - batch_delay: 5 - # Last Month - - start_days_ago: 30 - max_batch_events: 50 - batch_delay: 10 - # Last 3 months - - start_days_ago: 90 - max_batch_events: 100 - batch_delay: 10 - # The start of time - - start_days_ago: -1 - max_batch_events: 500 - batch_delay: 10 - # Should puppet avatars be fetched from the server even if an avatar is already set? - user_avatar_sync: true - # Should Matrix users leaving groups be bridged to WhatsApp? - bridge_matrix_leave: true - # Should the bridge update the m.direct account data event when double puppeting is enabled. - # Note that updating the m.direct event is not atomic (except with mautrix-asmux) - # and is therefore prone to race conditions. - sync_direct_chat_list: false - # Should the bridge use MSC2867 to bridge manual "mark as unread"s from - # WhatsApp and set the unread status on initial backfill? - # This will only work on clients that support the m.marked_unread or - # com.famedly.marked_unread room account data. - sync_manual_marked_unread: true - # When double puppeting is enabled, users can use `!wa toggle` to change whether - # presence is bridged. This setting sets the default value. - # Existing users won't be affected when these are changed. - default_bridge_presence: true - # Send the presence as "available" to whatsapp when users start typing on a portal. - # This works as a workaround for homeservers that do not support presence, and allows - # users to see when the whatsapp user on the other side is typing during a conversation. +{% raw %} + displayname_template: '{{or .BusinessName .PushName .Phone .RedactedPhone "Unknown user"}} (WA)' +{% endraw %} + + call_start_notices: true + identity_change_notices: false send_presence_on_typing: false - # Should the bridge always send "active" delivery receipts (two gray ticks on WhatsApp) - # even if the user isn't marked as online (e.g. when presence bridging isn't enabled)? - # - # By default, the bridge acts like WhatsApp web, which only sends active delivery - # receipts when it's in the foreground. - force_active_delivery_receipts: false - # Servers to always allow double puppeting from - double_puppet_server_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_SYNAPSE_URL }} - # Allow using double puppeting from any server with a valid client .well-known file. - double_puppet_allow_discovery: false - # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth - # - # If set, double puppeting will be enabled automatically for local users - # instead of users having to find an access token and run `login-matrix` - # manually. - login_shared_secret_map: - {{ MATRIX_SERVER_NAME }}: {{ MATRIX_REGISTRATION_SHARED_SECRET }} - # Whether to explicitly set the avatar and room name for private chat portal rooms. - # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms. - # If set to `always`, all DM rooms will have explicit names and avatars set. - # If set to `never`, DM rooms will never have names and avatars set. - private_chat_portal_meta: default - # Should group members be synced in parallel? This makes member sync faster - parallel_member_sync: false - # Should Matrix m.notice-type messages be bridged? - bridge_notices: true - # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run. - # This field will automatically be changed back to false after it, except if the config file is not writable. - resend_bridge_info: false - # When using double puppeting, should muted chats be muted in Matrix? - mute_bridging: false - # When using double puppeting, should archived chats be moved to a specific tag in Matrix? - # Note that WhatsApp unarchives chats when a message is received, which will also be mirrored to Matrix. - # This can be set to a tag (e.g. m.lowpriority), or null to disable. - archive_tag: null - # Same as above, but for pinned chats. The favorite tag is called m.favourite - pinned_tag: null - # Should mute status and tags only be bridged when the portal room is created? - tag_only_on_create: true - # Should WhatsApp status messages be bridged into a Matrix room? - # Disabling this won't affect already created status broadcast rooms. enable_status_broadcast: true - # Should sending WhatsApp status messages be allowed? - # This can cause issues if the user has lots of contacts, so it's disabled by default. disable_status_broadcast_send: true - # Should the status broadcast room be muted and moved into low priority by default? - # This is only applied when creating the room, the user can unmute it later. mute_status_broadcast: true - # Tag to apply to the status broadcast room. + pinned_tag: m.favourite + archive_tag: status_broadcast_tag: m.lowpriority - # Should the bridge use thumbnails from WhatsApp? - # They're disabled by default due to very low resolution. whatsapp_thumbnail: false - # Allow invite permission for user. User can invite any bots to room with whatsapp - # users (private chat and groups) - allow_user_invite: false - # Whether or not created rooms should have federation enabled. - # If false, created portal rooms will never be federated. - federate_rooms: true - # Should the bridge never send alerts to the bridge management room? - # These are mostly things like the user being logged out. - disable_bridge_alerts: false - # Should the bridge stop if the WhatsApp server says another user connected with the same session? - # This is only safe on single-user bridges. - crash_on_stream_replaced: false - # Should the bridge detect URLs in outgoing messages, ask the homeserver to generate a preview, - # and send it to WhatsApp? URL previews can always be sent using the `com.beeper.linkpreviews` - # key in the event content even if this is disabled. url_previews: false - # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552. - # This is currently not supported in most clients. - caption_in_message: false - # Send galleries as a single event? This is not an MSC (yet). - beeper_galleries: false - # Should polls be sent using MSC3381 event types? extev_polls: false - # Should cross-chat replies from WhatsApp be bridged? Most servers and clients don't support this. - cross_room_replies: false - # Disable generating reply fallbacks? Some extremely bad clients still rely on them, - # but they're being phased out and will be completely removed in the future. - disable_reply_fallbacks: false - # Maximum time for handling Matrix events. Duration strings formatted for https://pkg.go.dev/time#ParseDuration - # Null means there's no enforced timeout. - message_handling_timeout: - # Send an error message after this timeout, but keep waiting for the response until the deadline. - # This is counted from the origin_server_ts, so the warning time is consistent regardless of the source of delay. - # If the message is older than this when it reaches the bridge, the message won't be handled at all. - error_after: null - # Drop messages after this timeout. They may still go through if the message got sent to the servers. - # This is counted from the time the bridge starts handling the message. - deadline: 120s - - # The prefix for commands. Only required in non-management rooms. + disable_view_once: false + force_active_delivery_receipts: false + direct_media_auto_request: true + initial_auto_reconnect: true + use_whatsapp_retry_store: false + + animated_sticker: + target: webp + args: + width: 320 + height: 320 + fps: 25 + + history_sync: + max_initial_conversations: -1 + request_full_sync: false + dispatch_wait: 1m + full_sync_config: + days_limit: null + size_mb_limit: null + storage_quota_mb: null + media_requests: + auto_request_media: true + request_method: immediate + request_local_time: 120 + max_async_handle: 2 + backwards_on_demand: false + +bridge: command_prefix: "!wa" - - # Messages sent upon joining a management room. - # Markdown is supported. The defaults are listed below. - management_room_text: - # Sent when joining a room. - welcome: "Hello, I'm a WhatsApp bridge bot." - # Sent when joining a management room and the user is already logged in. - welcome_connected: "Use `help` for help." - # Sent when joining a management room and the user is not logged in. - welcome_unconnected: "Use `help` for help or `login` to log in." - # Optional extra text sent when joining a management room. - additional_help: "" - - # End-to-bridge encryption support options. - # - # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info. - encryption: - # Allow encryption, work in group chat rooms with e2ee enabled - allow: false - # Default to encryption, force-enable encryption in all portals the bridge creates - # This will cause the bridge bot to be in private chats for the encryption to work properly. - default: false - # Whether to use MSC2409/MSC3202 instead of /sync long polling for receiving encryption-related data. - appservice: false - # Require encryption, drop any unencrypted messages. - require: false - # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled. - # You must use a client that supports requesting keys from other users to use this feature. - allow_key_sharing: false - # Should users mentions be in the event wire content to enable the server to send push notifications? - plaintext_mentions: false - # Options for deleting megolm sessions from the bridge. - delete_keys: - # Beeper-specific: delete outbound sessions when hungryserv confirms - # that the user has uploaded the key to key backup. - delete_outbound_on_ack: false - # Don't store outbound sessions in the inbound table. - dont_store_outbound: false - # Ratchet megolm sessions forward after decrypting messages. - ratchet_on_decrypt: false - # Delete fully used keys (index >= max_messages) after decrypting messages. - delete_fully_used_on_decrypt: false - # Delete previous megolm sessions from same device when receiving a new one. - delete_prev_on_new_session: false - # Delete megolm sessions received from a device when the device is deleted. - delete_on_device_delete: false - # Periodically delete megolm sessions when 2x max_age has passed since receiving the session. - periodically_delete_expired: false - # Delete inbound megolm sessions that don't have the received_at field used for - # automatic ratcheting and expired session deletion. This is meant as a migration - # to delete old keys prior to the bridge update. - delete_outdated_inbound: false - # What level of device verification should be required from users? - # - # Valid levels: - # unverified - Send keys to all device in the room. - # cross-signed-untrusted - Require valid cross-signing, but trust all cross-signing keys. - # cross-signed-tofu - Require valid cross-signing, trust cross-signing keys on first use (and reject changes). - # cross-signed-verified - Require valid cross-signing, plus a valid user signature from the bridge bot. - # Note that creating user signatures from the bridge bot is not currently possible. - # verified - Require manual per-device verification - # (currently only possible by modifying the `trust` column in the `crypto_device` database table). - verification_levels: - # Minimum level for which the bridge should send keys to when bridging messages from WhatsApp to Matrix. - receive: unverified - # Minimum level that the bridge should accept for incoming Matrix messages. - send: unverified - # Minimum level that the bridge should require for accepting key requests. - share: cross-signed-tofu - # Options for Megolm room key rotation. These options allow you to - # configure the m.room.encryption event content. See: - # https://spec.matrix.org/v1.3/client-server-api/#mroomencryption for - # more information about that event. - rotation: - # Enable custom Megolm room key rotation settings. Note that these - # settings will only apply to rooms created after this option is - # set. - enable_custom: false - # The maximum number of milliseconds a session should be used - # before changing it. The Matrix spec recommends 604800000 (a week) - # as the default. - milliseconds: 604800000 - # The maximum number of messages that should be sent with a given a - # session before changing it. The Matrix spec recommends 100 as the - # default. - messages: 100 - # Disable rotating keys when a user's devices change? - # You should not enable this option unless you understand all the implications. - disable_device_change_key_rotation: false - - # Settings for provisioning API - provisioning: - # Prefix for the provisioning API paths. - prefix: /_matrix/provision - # Shared secret for authentication. If set to "generate", a random secret will be generated, - # or if set to "disable", the provisioning API will be disabled. - shared_secret: generate - # Enable debug API at /debug with provisioning authentication. - debug_endpoints: false - - # Permissions for using the bridge. - # Permitted values: - # relay - Talk through the relaybot (if enabled), no access otherwise - # user - Access to use the bridge to chat with a WhatsApp account. - # admin - User level and some additional administration tools - # Permitted keys: - # * - All Matrix users - # domain - All users on that homeserver - # mxid - Specific user + personal_filtering_spaces: false + private_chat_portal_meta: true + async_events: false + split_portals: false + resend_bridge_info: false + no_bridge_info_state_key: false + bridge_status_notices: errors + unknown_error_auto_reconnect: null + unknown_error_max_auto_reconnects: 10 + + bridge_matrix_leave: true + bridge_notices: true + tag_only_on_create: true + only_bridge_tags: [m.favourite, m.lowpriority] + mute_only_on_create: true + deduplicate_matrix_messages: false + cross_room_replies: false + revert_failed_state_changes: false + kick_matrix_users: true + enable_send_state_requests: false + phone_numbers_in_profile: false + + cleanup_on_logout: + enabled: false + manual: + private: nothing + relayed: nothing + shared_no_users: nothing + shared_has_users: nothing + bad_credentials: + private: nothing + relayed: nothing + shared_no_users: nothing + shared_has_users: nothing + + relay: + enabled: false + admin_only: true + prefer_default: true + allow_bridge: true + default_relays: [] + message_formats: +{% raw %} + m.text: "{{ .Sender.DisambiguatedName }}: {{ .Message }}" + m.notice: "{{ .Sender.DisambiguatedName }}: {{ .Message }}" + m.emote: "* {{ .Sender.DisambiguatedName }} {{ .Message }}" + m.file: "{{ .Sender.DisambiguatedName }} sent a file{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.image: "{{ .Sender.DisambiguatedName }} sent an image{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.audio: "{{ .Sender.DisambiguatedName }} sent an audio file{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.video: "{{ .Sender.DisambiguatedName }} sent a video{{ if .Caption }}: {{ .Caption }}{{ end }}" + m.location: "{{ .Sender.DisambiguatedName }} sent a location{{ if .Caption }}: {{ .Caption }}{{ end }}" +{% endraw %} + + portal_create_filter: + mode: deny + list: [] + always_deny_from_login: [] + permissions: "*": relay "{{ MATRIX_SERVER_NAME }}": user "@{{ MATRIX_ADMINISTRATOR_USERNAME }}:{{ MATRIX_SERVER_NAME }}": admin - - # Settings for relay mode - relay: - # Whether relay mode should be allowed. If allowed, `!wa set-relay` can be used to turn any - # authenticated user into a relaybot for that chat. - enabled: false - # Should only admins be allowed to set themselves as relay users? - admin_only: true - # The formats to use when sending messages to WhatsApp via the relaybot. - message_formats: + +database: + type: postgres + uri: postgres://mautrix_whatsapp_bridge:{{ lookup('config', application_id, 'credentials.mautrix_whatsapp_bridge_database_password') | regex_replace('/', '%2F') | regex_replace('@', '%40') | regex_replace(':', '%3A') }}@{{ MATRIX_POSTGRES_HOST }}/mautrix_whatsapp_bridge?sslmode=disable + max_open_conns: 20 + max_idle_conns: 2 + max_conn_idle_time: null + max_conn_lifetime: null + +homeserver: + address: http://synapse:{{ MATRIX_SYNAPSE_CONTAINER_PORT }} + domain: {{ MATRIX_SERVER_NAME }} + + software: standard + status_endpoint: + message_send_checkpoint_endpoint: + async_media: false + + websocket: false + ping_interval_seconds: 0 + retry_limit: 4 + +appservice: + address: http://mautrix-whatsapp:29318 + + hostname: 0.0.0.0 + port: 29318 + + id: whatsapp + bot: + username: whatsappbot + displayname: WhatsApp bridge bot + avatar: mxc://maunium.net/NeXNQarUbrlYBiPCpprYsRqr + + ephemeral_events: true + async_transactions: false + + as_token: "This value is generated when generating the registration" + hs_token: "This value is generated when generating the registration" + {% raw %} - m.text: "{{ .Sender.Displayname }}: {{ .Message }}" - m.notice: "{{ .Sender.Displayname }}: {{ .Message }}" - m.emote: "* {{ .Sender.Displayname }} {{ .Message }}" - m.file: "{{ .Sender.Displayname }} sent a file" - m.image: "{{ .Sender.Displayname }} sent an image" - m.audio: "{{ .Sender.Displayname }} sent an audio file" - m.video: "{{ .Sender.Displayname }} sent a video" - m.location: "{{ .Sender.Displayname }} sent a location" + username_template: whatsapp_{{.}} {% endraw %} -# Logging config. See https://github.com/tulir/zeroconfig for details. +matrix: + message_status_events: false + delivery_receipts: false + message_error_notices: true + sync_direct_chat_list: false + federate_rooms: true + upload_file_threshold: 5242880 + ghost_extra_profile_info: false + +analytics: + token: null + url: null + user_id: null + +provisioning: + shared_secret: generate + allow_matrix_auth: true + debug_endpoints: false + enable_session_transfers: false + +public_media: + enabled: false + signing_key: generate + expiry: 0 + hash_length: 32 + path_prefix: /_mautrix/publicmedia + use_database: false + +direct_media: + enabled: false + server_name: discord-media.example.com + well_known_response: + media_id_prefix: + allow_proxy: true + server_key: generate + +backfill: + enabled: true + max_initial_messages: 50 + max_catchup_messages: 500 + unread_hours_threshold: 720 + threads: + max_initial_messages: 50 + queue: + enabled: false + manual: false + batch_size: 100 + batch_delay: 20 + max_batches: -1 + max_batches_override: {} + +double_puppet: + servers: + {{ MATRIX_SERVER_NAME }}: {{ MATRIX_SYNAPSE_URL }} + allow_discovery: false + secrets: + {{ MATRIX_SERVER_NAME }}: {{ MATRIX_REGISTRATION_SHARED_SECRET }} + +encryption: + allow: false + default: false + require: false + appservice: false + msc4190: false + msc4392: false + self_sign: false + allow_key_sharing: false + plaintext_mentions: false + pickle_key: generate + delete_keys: + delete_outbound_on_ack: false + dont_store_outbound: false + ratchet_on_decrypt: false + delete_fully_used_on_decrypt: false + delete_prev_on_new_session: false + delete_on_device_delete: false + periodically_delete_expired: false + delete_outdated_inbound: false + verification_levels: + receive: unverified + send: unverified + share: cross-signed-tofu + rotation: + enable_custom: false + milliseconds: 604800000 + messages: 100 + disable_device_change_key_rotation: false + +env_config_prefix: null + logging: min_level: debug writers: - type: stdout - format: pretty-colored \ No newline at end of file + format: pretty-colored diff --git a/roles/web-app-matrix/templates/flavor/compose/services.yml.j2 b/roles/web-app-matrix/templates/flavor/compose/services.yml.j2 index 59f730de42..95fcf1b91d 100644 --- a/roles/web-app-matrix/templates/flavor/compose/services.yml.j2 +++ b/roles/web-app-matrix/templates/flavor/compose/services.yml.j2 @@ -10,7 +10,7 @@ - {{ MATRIX_SYNAPSE_CONFIG_PATH_HOST }}:{{ MATRIX_SYNAPSE_CONFIG_PATH_CONTAINER }}:ro - {{ MATRIX_SYNAPSE_LOG_PATH_HOST }}:{{ MATRIX_SYNAPSE_LOG_PATH_CONTAINER }}:ro {% for item in MATRIX_BRIDGES %} - - {{ lookup('container', application_id, 'directories.instance') }}mautrix/{{ item.bridge_name }}/registration.yaml:{{ MATRIX_REGISTRATION_FILE_FOLDER }}{{ item.bridge_name }}.registration.yaml:ro + - {{ lookup('container', application_id, 'directories.instance') }}mautrix/{{ item.bridge_name }}:{{ MATRIX_REGISTRATION_FILE_FOLDER }}mautrix-{{ item.bridge_name }}:ro {% endfor %} environment: - SYNAPSE_SERVER_NAME={{ MATRIX_SYNAPSE_DOMAIN }} @@ -18,13 +18,6 @@ ports: - "{{ DOCKER_BIND_HOST }}:{{lookup('config', 'web-app-matrix', 'services.synapse.ports.local.http')}}:{{ container_port }}" {% include 'roles/sys-svc-container/templates/healthcheck/curl.yml.j2' %} -{% if MATRIX_BRIDGES | length > 0 %} -{% for item in MATRIX_BRIDGES %} - mautrix-{{ item.bridge_name }}: - condition: service_healthy -{% endfor %} -{% else %} -{% endif %} {{ lookup('container_depends_on', application_id) }} {% include 'roles/sys-svc-container/templates/networks.yml.j2' %} element: @@ -48,9 +41,10 @@ - ./mautrix/{{ item.bridge_name }}:/data healthcheck: test: ["CMD-SHELL", "test -f {{ MATRIX_REGISTRATION_FILE_FOLDER }}registration.yaml || exit 1"] - interval: 1m + interval: 15s timeout: 10s - retries: 3 + retries: 5 + start_period: 5m {% include 'roles/sys-svc-container/templates/networks.yml.j2' %} {% endfor %} {% if lookup('config', application_id, 'services.matrix.plugins.chatgpt') | bool %} diff --git a/roles/web-app-matrix/templates/flavor/compose/synapse/homeserver.yaml.j2 b/roles/web-app-matrix/templates/flavor/compose/synapse/homeserver.yaml.j2 index 8f59142582..f049287c23 100644 --- a/roles/web-app-matrix/templates/flavor/compose/synapse/homeserver.yaml.j2 +++ b/roles/web-app-matrix/templates/flavor/compose/synapse/homeserver.yaml.j2 @@ -1,6 +1,6 @@ server_name: "{{ MATRIX_SERVER_NAME }}" pid_file: /data/homeserver.pid -max_upload_size: {{ webserver_client_max_body_size }} +max_upload_size: {{ webserver_client_max_body_size | default(lookup('config', application_id, 'server.client_max_body_size', '100m')) }} listeners: - port: {{ MATRIX_SYNAPSE_CONTAINER_PORT }} tls: false @@ -71,9 +71,9 @@ oidc_providers: backchannel_logout_enabled: true {% endif %} -{% if MATRIX_BRIDGES | bool %} +{% if MATRIX_BRIDGES | length > 0 %} app_service_config_files: {% for item in MATRIX_BRIDGES %} - - {{ MATRIX_REGISTRATION_FILE_FOLDER }}{{ item.bridge_name }}.registration.yaml + - {{ MATRIX_REGISTRATION_FILE_FOLDER }}mautrix-{{ item.bridge_name }}/registration.yaml {% endfor %} {% endif %} \ No newline at end of file diff --git a/roles/web-app-matrix/templates/playwright.env.j2 b/roles/web-app-matrix/templates/playwright.env.j2 index 373113327b..392b572afa 100644 --- a/roles/web-app-matrix/templates/playwright.env.j2 +++ b/roles/web-app-matrix/templates/playwright.env.j2 @@ -51,3 +51,7 @@ SEAWEEDFS_FILER_URL={{ ((lookup('tls', 'web-app-seaweedfs', 'protocols.web') ~ ' SEAWEEDFS_SERVICE_ENABLED={{ lookup('config', application_id, 'services.seaweedfs.enabled') | string | lower }} SSO_SERVICE_ENABLED={{ lookup('config', application_id, 'services.sso.enabled') | string | lower }} + +{% for bridge in MATRIX_BRIDGES | default([], true) %} +MAUTRIX_{{ bridge.bridge_name | upper | regex_replace('[^A-Z0-9]+', '_') }}_ADDON_ENABLED=true +{% endfor %} diff --git a/roles/web-app-matrix/vars/bridges.yml b/roles/web-app-matrix/vars/bridges.yml deleted file mode 100644 index 8b70825fd1..0000000000 --- a/roles/web-app-matrix/vars/bridges.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -bridges_configuration: - - database_password: "{{ lookup('config', application_id, 'credentials.mautrix_whatsapp_bridge_database_password') }}" - database_username: "mautrix_whatsapp_bridge" - database_name: "mautrix_whatsapp_bridge" - bridge_name: "whatsapp" - - - database_password: "{{ lookup('config', application_id, 'credentials.mautrix_telegram_bridge_database_password') }}" - database_username: "mautrix_telegram_bridge" - database_name: "mautrix_telegram_bridge" - bridge_name: "telegram" - - - database_password: "{{ lookup('config', application_id, 'credentials.mautrix_signal_bridge_database_password') }}" - database_username: "mautrix_signal_bridge" - database_name: "mautrix_signal_bridge" - bridge_name: "signal" - - - database_password: "{{ lookup('config', application_id, 'credentials.mautrix_slack_bridge_database_password') }}" - database_username: "mautrix_slack_bridge" - database_name: "mautrix_slack_bridge" - bridge_name: "slack" - - - database_password: "{{ lookup('config', application_id, 'credentials.mautrix_facebook_bridge_database_password') }}" - database_username: "mautrix_facebook_bridge" - database_name: "mautrix_facebook_bridge" - bridge_name: "facebook" - - - database_password: "{{ lookup('config', application_id, 'credentials.mautrix_instagram_bridge_database_password') }}" - database_username: "mautrix_instagram_bridge" - database_name: "mautrix_instagram_bridge" - bridge_name: "instagram" diff --git a/roles/web-app-matrix/vars/main.yml b/roles/web-app-matrix/vars/main.yml index 83c96d363b..4905bedf4a 100644 --- a/roles/web-app-matrix/vars/main.yml +++ b/roles/web-app-matrix/vars/main.yml @@ -2,7 +2,6 @@ application_id: "web-app-matrix" location_upload: "~ ^/_matrix/media/v3/" -webserver_client_max_body_size: "{{ lookup('config', application_id, 'server.client_max_body_size') }}" MATRIX_SERVER_NAME: "{{ lookup('config', application_id, 'services.matrix.server_name') }}" MATRIX_ADMINISTRATOR_USERNAME: "{{ lookup('users', 'administrator').username }}" diff --git a/roles/web-app-mattermost/README.md b/roles/web-app-mattermost/README.md index e2091e5185..6acef659b9 100644 --- a/roles/web-app-mattermost/README.md +++ b/roles/web-app-mattermost/README.md @@ -38,6 +38,10 @@ Key settings in `meta/services.yml` and `meta/server.yml`: | `services.mattermost.version` | `latest` | Docker image tag | | `server.domains.canonical` | `mattermost.{{ DOMAIN_PRIMARY }}` | Public domain | +## Addons + +This role declares no addons (it ships no `meta/addons/` directory). Mattermost **Team Edition** manages plugin install and enablement at runtime; there is no declarative per-plugin install path in this role. Plugins are operator-managed and persisted through the named plugin volumes (`plugins`, `client-plugins`) declared in `meta/volumes.yml`. No addon bridges any in-repo service. + ## References - [Mattermost Docker Install](https://docs.mattermost.com/install/install-docker.html) diff --git a/roles/web-app-mattermost/meta/services.yml b/roles/web-app-mattermost/meta/services.yml index 5d23e1d397..aad1e4b7f8 100644 --- a/roles/web-app-mattermost/meta/services.yml +++ b/roles/web-app-mattermost/meta/services.yml @@ -1,30 +1,41 @@ --- sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" # nocheck: dynamic-flag ldap: enabled: false dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" +# nocheck: playwright-service-flag: shared provider entity; the app itself is covered by this role's own Playwright spec mattermost: + bond: 1 + enabled: true + shared: true + user_access_token_enabled: "{{ 'web-app-nextcloud' in group_names }}" image: mattermost/mattermost-team-edition - version: 11.8.1 + version: 11.8.2 min_storage: 10GB backup: no_stop_required: true @@ -51,19 +62,23 @@ minio: enabled: false # nocheck: dynamic-flag shared: false # nocheck: dynamic-flag seaweedfs: + bond: 1 enabled: "{{ 'web-app-seaweedfs' in group_names }}" shared: "{{ 'web-app-seaweedfs' in group_names }}" # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag # nocheck: playwright-service-flag โ€” canonical:cdn alias of static-asset provider javascript: + bond: 1 enabled: true shared: true # nocheck: native-metrics โ€” fragment ships prepared; operator flips `native_metrics.enabled` to opt in prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" native_metrics: diff --git a/roles/web-app-mattermost/tasks/01_core.yml b/roles/web-app-mattermost/tasks/01_core.yml new file mode 100644 index 0000000000..291ad997a6 --- /dev/null +++ b/roles/web-app-mattermost/tasks/01_core.yml @@ -0,0 +1,21 @@ +--- +- include_tasks: utils/once/flag.yml + +- name: "Remove stale nginx config for renamed domain chat.{{ DOMAIN_PRIMARY }}" # nocheck: domain-spot + ansible.builtin.file: + path: "{{ [lookup('nginx', 'directories.configuration.servers'), 'https', 'chat.' ~ DOMAIN_PRIMARY ~ '.conf'] | path_join }}" + state: absent + notify: restart openresty + +- name: "load docker, db and proxy for {{ application_id }}" + include_role: + name: sys-stk-full + vars: + compose_handlers_flush: true + +- name: "Run Mattermost setup routines" + include_tasks: 01_setup.yml + +- name: "Configure Keycloak OIDC mappers for Mattermost" + include_tasks: 02_keycloak.yml + when: MATTERMOST_OIDC_ENABLED | bool diff --git a/roles/web-app-mattermost/tasks/main.yml b/roles/web-app-mattermost/tasks/main.yml index 857edd032a..7a09e6789e 100644 --- a/roles/web-app-mattermost/tasks/main.yml +++ b/roles/web-app-mattermost/tasks/main.yml @@ -1,19 +1,6 @@ --- -- name: "Remove stale nginx config for renamed domain chat.{{ DOMAIN_PRIMARY }}" # nocheck: domain-spot - ansible.builtin.file: - path: "{{ [lookup('nginx', 'directories.configuration.servers'), 'https', 'chat.' ~ DOMAIN_PRIMARY ~ '.conf'] | path_join }}" - state: absent - notify: restart openresty - -- name: "load docker, db and proxy for {{ application_id }}" - include_role: - name: sys-stk-full +- name: "Execute Role (once)" + include_tasks: 01_core.yml + when: run_once_web_app_mattermost is not defined vars: - compose_handlers_flush: true - -- name: "Run Mattermost setup routines" - include_tasks: 01_setup.yml - -- name: "Configure Keycloak OIDC mappers for Mattermost" - include_tasks: 02_keycloak.yml - when: MATTERMOST_OIDC_ENABLED | bool + application_id: "web-app-mattermost" diff --git a/roles/web-app-mattermost/templates/env.j2 b/roles/web-app-mattermost/templates/env.j2 index faf5717283..ec7f094e35 100644 --- a/roles/web-app-mattermost/templates/env.j2 +++ b/roles/web-app-mattermost/templates/env.j2 @@ -6,6 +6,7 @@ MM_SERVICESETTINGS_SITEURL={{ lookup('tls', application_id, 'url.base') }} MM_SERVICESETTINGS_LISTENADDRESS=:{{ container_port }} MM_SERVICESETTINGS_ALLOWCORSFROM={{ lookup('tls', application_id, 'url.base') }} MM_SERVICESETTINGS_ENABLELOCALMODE=true +MM_SERVICESETTINGS_ENABLEUSERACCESSTOKENS={{ lookup('config', application_id, 'services.mattermost.user_access_token_enabled') }} MM_EXPERIMENTALSETTINGS_DISABLELANDINGPAGE=true # Logging diff --git a/roles/web-app-mediawiki/README.md b/roles/web-app-mediawiki/README.md index 65da290a65..db63fac264 100644 --- a/roles/web-app-mediawiki/README.md +++ b/roles/web-app-mediawiki/README.md @@ -16,6 +16,15 @@ This role deploys MediaWiki using Docker, automating the setup of your wiki inst - **Scalable Deployment:** Utilize Docker for a portable and scalable setup that adapts as your community grows. - **Secure and Reliable:** Benefit from secure access via an NGINX reverse proxy combined with a MariaDB backend for reliable data storage. +## Addons + +This role ships its OIDC login stack as unified addons declared in [`meta/addons/`](meta/addons/). Both are MediaWiki extensions installed from upstream and gated on the `sso` service flag (`web-app-keycloak` co-deployed). The OIDC client secret is rendered through `templates/oidc.php.j2` and never inlined into the addon declaration. + +| Addon | Mechanism | Default state | Bridges | +|---|---|---|---| +| PluggableAuth | extension | enabled when `services.sso.enabled` | none | +| OpenIDConnect | extension | enabled when `services.sso.enabled` | `sso` | + ## Further Resources - [MediaWiki Official Website](https://www.mediawiki.org/) diff --git a/roles/web-app-mediawiki/files/playwright/addons/OpenIDConnect.spec.js b/roles/web-app-mediawiki/files/playwright/addons/OpenIDConnect.spec.js new file mode 100644 index 0000000000..d613a8677e --- /dev/null +++ b/roles/web-app-mediawiki/files/playwright/addons/OpenIDConnect.spec.js @@ -0,0 +1,110 @@ +const { test, expect } = require("@playwright/test"); + +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const { skipUnlessServiceEnabled } = require("../service-gating"); +const { + decodeDotenvQuotedValue, + normalizeBaseUrl, + runAdminFlow, +} = require("../personas"); + +test.use({ ignoreHTTPSErrors: true }); + +const appBaseUrl = normalizeBaseUrl(process.env.APP_BASE_URL || ""); +const canonicalDomain = decodeDotenvQuotedValue(process.env.CANONICAL_DOMAIN || ""); + +test("OpenIDConnect: wiki login hands off to the Keycloak OIDC authorize endpoint", async ({ + page, +}) => { + skipUnlessAddonEnabled("OpenIDConnect"); + skipUnlessServiceEnabled("sso"); + + expect(appBaseUrl, "APP_BASE_URL must be set").toBeTruthy(); + expect(canonicalDomain, "CANONICAL_DOMAIN must be set").toBeTruthy(); + + await page.context().clearCookies(); + + // Drive MediaWiki's PluggableAuth login entry directly. With OpenIDConnect + // loaded and its provider wired to Keycloak (oidc.php -> $wgPluggableAuth_Config + // providerURL/clientID), PluggableAuth must redirect the browser to the + // Keycloak OIDC authorization endpoint (.../protocol/openid-connect/auth?...). + // A wiki where OpenIDConnect is NOT actually wired would instead stay on a + // local MediaWiki login form and never reach the IdP authorize endpoint, so + // this assertion fails the integration loudly. + const oidcAuthorizeUrl = await new Promise((resolve) => { + let settled = false; + const done = (value) => { + if (settled) return; + settled = true; + resolve(value); + }; + + page.on("request", (request) => { + if (/\/protocol\/openid-connect\/auth(\?|$)/.test(request.url())) { + done(request.url()); + } + }); + + page + .goto(`${appBaseUrl}/index.php?title=Special:UserLogin&returnto=Main+Page`, { + waitUntil: "domcontentloaded", + timeout: 60_000, + }) + .then(() => page.waitForLoadState("networkidle", { timeout: 30_000 }).catch(() => {})) + .then(() => { + if (/\/protocol\/openid-connect\/auth/.test(page.url())) { + done(page.url()); + return; + } + done(null); + }) + .catch(() => done(null)); + }); + + expect( + oidcAuthorizeUrl, + "MediaWiki login did NOT redirect to the Keycloak OIDC authorize endpoint " + + "(/protocol/openid-connect/auth) โ€” OpenIDConnect is not wired to the Keycloak provider. " + + `Current URL: ${page.url()}.`, + ).toBeTruthy(); + + // The authorize endpoint must be an absolute Keycloak URL on a DIFFERENT host + // than the wiki itself โ€” proves the provider URL points at the real partner + // IdP (a separate Keycloak service), not a wiki-local stub or relative path. + const authorizeHost = new URL(oidcAuthorizeUrl).host; + const wikiHost = new URL(appBaseUrl).host; + expect( + authorizeHost && authorizeHost !== wikiHost, + "OIDC authorize endpoint is not served from a distinct Keycloak IdP host " + + `(wiki host ${wikiHost}, authorize URL ${oidcAuthorizeUrl}).`, + ).toBe(true); + + // Full coupling: complete the OIDC round-trip through Keycloak and assert we + // land on an authenticated MediaWiki surface (personal/logout tools that only + // render once OpenIDConnect has authenticated the user). + await page.context().clearCookies(); + await runAdminFlow(page, { + adminInteraction: async (interactivePage) => { + const wikiSurface = interactivePage + .locator("#mw-content-text") + .or(interactivePage.locator("#firstHeading")) + .or(interactivePage.locator("#mw-panel")) + .or(interactivePage.locator("body.mediawiki")) + .first(); + await expect(wikiSurface).toBeVisible({ timeout: 60_000 }); + + const authenticatedMarker = interactivePage + .locator("#pt-logout") + .or(interactivePage.locator("#pt-userpage")) + .or(interactivePage.locator("#p-personal a[href*='Logout' i]")) + .or(interactivePage.locator("#p-personal a[href*='Special:UserLogout' i]")) + .or( + interactivePage.getByRole("link", { + name: /log\s*out|sign\s*out|abmelden/i, + }), + ) + .first(); + await expect(authenticatedMarker).toBeVisible({ timeout: 60_000 }); + }, + }); +}); diff --git a/roles/web-app-mediawiki/files/playwright/addons/PluggableAuth.spec.js b/roles/web-app-mediawiki/files/playwright/addons/PluggableAuth.spec.js new file mode 100644 index 0000000000..3329169138 --- /dev/null +++ b/roles/web-app-mediawiki/files/playwright/addons/PluggableAuth.spec.js @@ -0,0 +1,91 @@ +const { test, expect } = require("@playwright/test"); + +const { skipUnlessAddonEnabled } = require("../addon-gating"); +const { skipUnlessServiceEnabled } = require("../service-gating"); +const { decodeDotenvQuotedValue, normalizeBaseUrl } = require("../personas"); + +test.use({ ignoreHTTPSErrors: true }); + +const appBaseUrl = normalizeBaseUrl(process.env.APP_BASE_URL || ""); +const canonicalDomain = decodeDotenvQuotedValue(process.env.CANONICAL_DOMAIN || ""); + +// PluggableAuth is the auth framework backing OpenIDConnect. The role wires it +// with EnableAutoLogin=true / EnableLocalLogin=false, so a wiki where the +// framework is actually loaded AND bound to the OIDC plugin must NOT serve the +// stock local username/password login form on Special:UserLogin โ€” it must +// instead hand off to the OIDC issuer (Keycloak) or render only the +// PluggableAuth SSO affordance. This test fails if PluggableAuth is absent or +// not coupled to the IdP. +test("PluggableAuth: framework replaces local login and is bound to the OIDC issuer", async ({ + page, +}) => { + skipUnlessAddonEnabled("PluggableAuth"); + skipUnlessServiceEnabled("sso"); + + expect(appBaseUrl, "APP_BASE_URL must be set").toBeTruthy(); + expect(canonicalDomain, "CANONICAL_DOMAIN must be set").toBeTruthy(); + + await page.context().clearCookies(); + + const loginUrl = `${appBaseUrl}/index.php?title=Special:UserLogin`; + await page.goto(loginUrl, { waitUntil: "domcontentloaded", timeout: 60_000 }); + + // Let an EnableAutoLogin handoff to the IdP settle. + await page + .waitForLoadState("networkidle", { timeout: 60_000 }) + .catch(() => {}); + + const finalUrl = page.url(); + const redirectedAway = !finalUrl.includes("Special:UserLogin"); + + if (redirectedAway) { + // EnableAutoLogin handed off to the configured OIDC issuer. Prove the + // landing page is genuinely an OAuth2/OIDC authorization surface coupled to + // this wiki, not an arbitrary redirect: the OIDC client sends the wiki's + // redirect_uri back to itself and Keycloak renders a recognizable login. + const url = new URL(finalUrl); + const looksLikeAuthEndpoint = + /\/(auth|authorize|protocol\/openid-connect\/auth|oauth2?\/authorize)/i.test( + url.pathname, + ) || url.searchParams.has("redirect_uri"); + + expect( + looksLikeAuthEndpoint, + `auto-login should land on the OIDC authorization endpoint, got ${finalUrl}`, + ).toBeTruthy(); + + // The OAuth request must carry this wiki as the client redirect target, + // proving the clientID/providerURL coupling from $wgPluggableAuth_Config. + const redirectUri = url.searchParams.get("redirect_uri") || ""; + if (redirectUri) { + expect( + decodeURIComponent(redirectUri), + "OIDC redirect_uri must point back at this wiki", + ).toContain(canonicalDomain); + } + + await expect(page.locator("body")).toContainText( + /sign in|log ?in|username|password|keycloak|openid/i, + { timeout: 60_000 }, + ); + return; + } + + // No auto-redirect: we are still on Special:UserLogin. PluggableAuth with + // EnableLocalLogin=false must have suppressed the stock local credential form + // and exposed only the SSO affordance bound to OIDC. + const body = page.locator("body"); + + await expect(body).toContainText( + /single sign[- ]?on|openid|\bsso\b|log ?in with|continue with|keycloak/i, + { timeout: 60_000 }, + ); + + // Stock MediaWiki local login has a password input named "wpPassword". With + // EnableLocalLogin=false the framework removes it; its presence would mean + // PluggableAuth is not actually driving the login surface. + await expect( + page.locator('input[name="wpPassword"]'), + "local password field must be absent when PluggableAuth owns login", + ).toHaveCount(0); +}); diff --git a/roles/web-app-mediawiki/meta/addons/OpenIDConnect.yml b/roles/web-app-mediawiki/meta/addons/OpenIDConnect.yml new file mode 100644 index 0000000000..4f51075479 --- /dev/null +++ b/roles/web-app-mediawiki/meta/addons/OpenIDConnect.yml @@ -0,0 +1,11 @@ +--- +enabled: "{{ lookup('config', 'web-app-mediawiki', 'services.sso.enabled') | bool }}" +required: false +mechanism: extension +source: upstream +bridges: + - sso +update: + monitored: true + catalog: mediawiki-extension + upstream_id: OpenIDConnect diff --git a/roles/web-app-mediawiki/meta/addons/PluggableAuth.yml b/roles/web-app-mediawiki/meta/addons/PluggableAuth.yml new file mode 100644 index 0000000000..575f55fef8 --- /dev/null +++ b/roles/web-app-mediawiki/meta/addons/PluggableAuth.yml @@ -0,0 +1,9 @@ +--- +enabled: "{{ lookup('config', 'web-app-mediawiki', 'services.sso.enabled') | bool }}" +required: false +mechanism: extension +source: upstream +update: + monitored: true + catalog: mediawiki-extension + upstream_id: PluggableAuth diff --git a/roles/web-app-mediawiki/meta/services.yml b/roles/web-app-mediawiki/meta/services.yml index 11cef82eb4..8d4def3165 100644 --- a/roles/web-app-mediawiki/meta/services.yml +++ b/roles/web-app-mediawiki/meta/services.yml @@ -1,22 +1,29 @@ --- # nocheck: seaweedfs-required โ€” object storage not implemented for this role sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" +# nocheck: playwright-service-flag โ€” DB engine, no Playwright surface (role-local integration tests) mariadb: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-mariadb' in group_names }}" mediawiki: @@ -39,9 +46,12 @@ mediawiki: mem_reservation: 256m mem_limit: 1g pids_limit: 512 +# nocheck: playwright-service-flag โ€” CSS consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-mediawiki/tasks/addons/OpenIDConnect.yml b/roles/web-app-mediawiki/tasks/addons/OpenIDConnect.yml new file mode 100644 index 0000000000..672ebd18a5 --- /dev/null +++ b/roles/web-app-mediawiki/tasks/addons/OpenIDConnect.yml @@ -0,0 +1,2 @@ +--- +# No exception logic required: the generic config SPOT (meta/addons/.yml plugin_configuration + tasks/plugin) fully wires this integration; coupling is verified by the Playwright spec. diff --git a/roles/web-app-mediawiki/tasks/addons/PluggableAuth.yml b/roles/web-app-mediawiki/tasks/addons/PluggableAuth.yml new file mode 100644 index 0000000000..672ebd18a5 --- /dev/null +++ b/roles/web-app-mediawiki/tasks/addons/PluggableAuth.yml @@ -0,0 +1,2 @@ +--- +# No exception logic required: the generic config SPOT (meta/addons/.yml plugin_configuration + tasks/plugin) fully wires this integration; coupling is verified by the Playwright spec. diff --git a/roles/web-app-mediawiki/templates/playwright.env.j2 b/roles/web-app-mediawiki/templates/playwright.env.j2 new file mode 100644 index 0000000000..54d558bb6c --- /dev/null +++ b/roles/web-app-mediawiki/templates/playwright.env.j2 @@ -0,0 +1,31 @@ +ADMIN_PASSWORD={{ lookup('users', 'administrator').password | dotenv_quote }} + +ADMIN_USERNAME={{ lookup('users', 'administrator').username | dotenv_quote }} + +APP_BASE_URL={{ lookup('tls', application_id, 'url.base') }} + +BIBER_PASSWORD={{ lookup('users', 'biber').password | dotenv_quote }} + +BIBER_USERNAME={{ lookup('users', 'biber').username | dotenv_quote }} + +CANONICAL_DOMAIN={{ lookup('domain', application_id) | dotenv_quote }} + +# nocheck: playwright-service-gate โ€” covered by sys-svc-mail integration tests, not the persona surface +EMAIL_SERVICE_ENABLED={{ lookup('config', application_id, 'services.email.enabled') | string | lower }} + +LOGOUT_SERVICE_ENABLED={{ lookup('config', application_id, 'services.logout.enabled') | string | lower }} + +MATOMO_BASE_URL={{ lookup('tls', 'web-app-matomo', 'url.base') | dotenv_quote }} + +MATOMO_SERVICE_ENABLED={{ lookup('config', application_id, 'services.matomo.enabled') | string | lower }} + +# Playwright has no custom ENTRYPOINT, so the infinito CA cert is not injected and npm fails on the proxy-intercepted npm registry TLS. +NODE_TLS_REJECT_UNAUTHORIZED=0 + +PERSONA_ADMINISTRATOR_BLOCKED=true + +PERSONA_BIBER_BLOCKED=true + +SSO_SERVICE_ENABLED={{ lookup('config', application_id, 'services.sso.enabled') | string | lower }} + +{{ lookup('addon_env_flags', application_id) }} diff --git a/roles/web-app-mediawiki/vars/main.yml b/roles/web-app-mediawiki/vars/main.yml index 953c0c083a..93576eb5c9 100644 --- a/roles/web-app-mediawiki/vars/main.yml +++ b/roles/web-app-mediawiki/vars/main.yml @@ -51,10 +51,13 @@ MEDIAWIKI_OIDC_LOCALLOGIN: false # Extensions MEDIAWIKI_EXT_BRANCH: "REL{{ (MEDIAWIKI_VERSION | string).split('.')[:2] | join('_') }}" MEDIAWIKI_EXT_CFG_BASE: "{{ [ MEDIAWIKI_CONFIG_DIR, 'mwext', MEDIAWIKI_EXT_BRANCH ] | path_join }}" -MEDIAWIKI_EXT_LIST: - - name: "PluggableAuth" - # nocheck: url - url: "{{ [ 'https://codeload.github.com/wikimedia/mediawiki-extensions-PluggableAuth/tar.gz/refs/heads/',MEDIAWIKI_EXT_BRANCH ] | url_join }}" - - name: "OpenIDConnect" - # nocheck: url - url: "{{ [ 'https://codeload.github.com/wikimedia/mediawiki-extensions-OpenIDConnect/tar.gz/refs/heads/',MEDIAWIKI_EXT_BRANCH ] | url_join }}" +MEDIAWIKI_EXT_NAMES: "{{ lookup('config', application_id, 'addons').keys() | list }}" +# nocheck: url +MEDIAWIKI_EXT_URL_PREFIX: "https://codeload.github.com/wikimedia/mediawiki-extensions-" +MEDIAWIKI_EXT_URL_SUFFIX: "/tar.gz/refs/heads/{{ MEDIAWIKI_EXT_BRANCH }}" +MEDIAWIKI_EXT_LIST: >- + {% set _exts = [] %} + {%- for _name in MEDIAWIKI_EXT_NAMES -%} + {%- set _ = _exts.append({'name': _name, 'url': MEDIAWIKI_EXT_URL_PREFIX ~ _name ~ MEDIAWIKI_EXT_URL_SUFFIX}) -%} + {%- endfor -%} + {{ _exts }} diff --git a/roles/web-app-mig/meta/services.yml b/roles/web-app-mig/meta/services.yml index e9bdffa31d..8cb1cfac3d 100644 --- a/roles/web-app-mig/meta/services.yml +++ b/roles/web-app-mig/meta/services.yml @@ -11,9 +11,11 @@ sso: enabled: false shared: false dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" # nocheck: dynamic-flag @@ -39,8 +41,10 @@ mig: pids_limit: 256 # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-mini-qr/meta/services.yml b/roles/web-app-mini-qr/meta/services.yml index eeb19e2fb4..9f9575a0b7 100644 --- a/roles/web-app-mini-qr/meta/services.yml +++ b/roles/web-app-mini-qr/meta/services.yml @@ -22,9 +22,11 @@ sso: enabled: false shared: false dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" # nocheck: dynamic-flag @@ -33,8 +35,10 @@ redis: disabled: true enabled: false css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-minio/meta/services.yml b/roles/web-app-minio/meta/services.yml index 6f719aab2f..0ee19f8733 100644 --- a/roles/web-app-minio/meta/services.yml +++ b/roles/web-app-minio/meta/services.yml @@ -1,22 +1,28 @@ --- sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" # nocheck: playwright-service-flag โ€” central object-store provider entity minio: + bond: 1 enabled: true shared: true backup: @@ -47,6 +53,7 @@ redis: enabled: false # nocheck: playwright-service-flag โ€” css consumer, no Playwright surface (provider: web-svc-css) css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" # nocheck: dynamic-flag @@ -56,9 +63,11 @@ javascript: shared: true # nocheck: playwright-service-flag โ€” AI backend, infrastructural ollama: + bond: 1 enabled: "{{ 'svc-ai-ollama' in group_names }}" shared: "{{ 'svc-ai-ollama' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" api: @@ -83,3 +92,7 @@ console: client: image: quay.io/minio/mc version: "RELEASE.2025-08-13T08-35-41Z" # nocheck: docker-version + cpus: "0.2" + mem_reservation: 128m + mem_limit: 256m + pids_limit: 256 diff --git a/roles/web-app-mobilizon/meta/services.yml b/roles/web-app-mobilizon/meta/services.yml index d05a5d0ef6..cfc01a0fdf 100644 --- a/roles/web-app-mobilizon/meta/services.yml +++ b/roles/web-app-mobilizon/meta/services.yml @@ -1,22 +1,28 @@ --- # nocheck: seaweedfs-playwright, seaweedfs-required โ€” Mobilizon (mainline, incl. 5.2.3) ships only Mobilizon.Web.Upload.Uploader.Local (no ex_aws/Uploader.S3 backend per upstream docs), so uploads cannot be routed to the object store and no S3 write surface exists to drive sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" postgres: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-postgres' in group_names }}" extensions: @@ -46,8 +52,10 @@ seaweedfs: enabled: false shared: false css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-moodle/meta/services.yml b/roles/web-app-moodle/meta/services.yml index 18197aa082..340474f15d 100644 --- a/roles/web-app-moodle/meta/services.yml +++ b/roles/web-app-moodle/meta/services.yml @@ -1,29 +1,40 @@ --- # nocheck: seaweedfs-required โ€” object storage not implemented for this role sso: + bond: 1 enabled: "{{ 'web-app-keycloak' in group_names }}" shared: "{{ 'web-app-keycloak' in group_names }}" flavor: oidc logout: + bond: 1 enabled: "{{ 'web-svc-logout' in group_names }}" shared: "{{ 'web-svc-logout' in group_names }}" dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" email: + bond: 1 enabled: "{{ 'web-app-mailu' in group_names }}" shared: "{{ 'web-app-mailu' in group_names }}" ldap: + bond: 1 enabled: "{{ 'svc-db-openldap' in group_names }}" shared: "{{ 'svc-db-openldap' in group_names }}" # nocheck: playwright-service-flag โ€” DB engine, covered by role-local integration tests mariadb: + bond: 1 enabled: true # nocheck: dynamic-flag shared: "{{ 'svc-db-mariadb' in group_names }}" +# nocheck: playwright-service-flag: shared provider entity; the app itself is covered by this role's own Playwright spec moodle: + bond: 1 + enabled: true + shared: true # Self-built image. The `image`+`version` pair is the PHP base the # Dockerfile builds FROM (php:8.3-fpm) โ€” this is also what GHCR # image-mirror discovery picks up, so the base stays cached. @@ -71,8 +82,10 @@ cron: pids_limit: 256 # nocheck: playwright-service-flag โ€” verified by web-svc-css provider spec css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" diff --git a/roles/web-app-moodle/tasks/01_core.yml b/roles/web-app-moodle/tasks/01_core.yml new file mode 100644 index 0000000000..526d3096bf --- /dev/null +++ b/roles/web-app-moodle/tasks/01_core.yml @@ -0,0 +1,93 @@ +--- +- include_tasks: utils/once/flag.yml + +- name: "load docker, db and proxy for {{ application_id }}" + include_role: + name: sys-stk-full + vars: + compose_handlers_flush: false + +- name: "Populate Dockerfile build context (entrypoint, config.php, install scripts, php conf)" + include_tasks: 01_build_context.yml + +- name: "Ensure nginx vhost dir exists in compose dir" + ansible.builtin.file: + path: "{{ docker_compose.directories.instance | default(lookup('container', application_id, 'directories.instance')) }}" + state: directory + mode: "0755" + +- name: "Render nginx vhost into compose dir" + ansible.builtin.template: + src: "nginx-moodle.conf.j2" + dest: "{{ docker_compose.directories.instance | default(lookup('container', application_id, 'directories.instance')) }}/nginx.conf" + mode: "0640" + notify: compose up + +- name: flush docker service + meta: flush_handlers + +- name: Wait for Moodle container to become healthy + block: + - name: Wait until the Moodle container is healthy + ansible.builtin.shell: > + container inspect --format '{% raw %}{{.State.Health.Status}}{% endraw %}' {{ MOODLE_CONTAINER }} + register: health_check + until: health_check.stdout.strip() == "healthy" + retries: 120 + delay: 5 + + rescue: + - name: Show Moodle container status + ansible.builtin.shell: > + container inspect {{ MOODLE_CONTAINER }} + register: moodle_inspect + changed_when: false + failed_when: false + + - name: Show Moodle container logs + ansible.builtin.shell: > + container logs --tail=200 {{ MOODLE_CONTAINER }} + register: moodle_logs + changed_when: false + failed_when: false + + - name: Fail with diagnostic output + ansible.builtin.fail: + msg: | + โŒ Moodle container did not become healthy in time. + + --- container inspect --- + {{ moodle_inspect.stdout | default('no output') }} + + --- container logs (last 200 lines) --- + {{ moodle_logs.stdout | default('no output') }} + +- name: "Run Moodle install (idempotent)" + command: > + container exec --user {{ MOODLE_RUNTIME_USER }} {{ MOODLE_CONTAINER }} + php {{ MOODLE_CLI_DIR }}/install_database.php + --agree-license + --adminuser={{ lookup('config', application_id, 'users.administrator.username') }} + --adminpass={{ lookup('config', application_id, 'credentials.user_password') | quote }} + --adminemail={{ lookup('config', application_id, 'users.administrator.email') }} + --fullname="{{ MOODLE_SITE_TITLE }}" + --shortname=Academy + register: moodle_install + changed_when: "'Moodle has been installed' in moodle_install.stdout" + failed_when: + - moodle_install.rc != 0 + - "'Moodle is already installed' not in moodle_install.stderr" + - "'already installed' not in moodle_install.stdout" + - "'Database tables already present' not in moodle_install.stderr" + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "Include ownership settings tasks for moodle" + include_tasks: 02_ownership.yml + +- name: "Configure OIDC login for Moodle (variant 0 only)" + include_tasks: 03_oidc.yml + when: MOODLE_OIDC_ENABLED | bool + +- name: "Configure auth_ldap (sync-only in variant 0, primary login in variant 1)" + include_tasks: 04_ldap_sync.yml + when: MOODLE_LDAP_ENABLED | bool diff --git a/roles/web-app-moodle/tasks/main.yml b/roles/web-app-moodle/tasks/main.yml index a1b3e991ff..131198ab80 100644 --- a/roles/web-app-moodle/tasks/main.yml +++ b/roles/web-app-moodle/tasks/main.yml @@ -1,91 +1,6 @@ --- -- name: "load docker, db and proxy for {{ application_id }}" - include_role: - name: sys-stk-full +- name: "Execute Role (once)" + include_tasks: 01_core.yml + when: run_once_web_app_moodle is not defined vars: - compose_handlers_flush: false - -- name: "Populate Dockerfile build context (entrypoint, config.php, install scripts, php conf)" - include_tasks: 01_build_context.yml - -- name: "Ensure nginx vhost dir exists in compose dir" - ansible.builtin.file: - path: "{{ docker_compose.directories.instance | default(lookup('container', application_id, 'directories.instance')) }}" - state: directory - mode: "0755" - -- name: "Render nginx vhost into compose dir" - ansible.builtin.template: - src: "nginx-moodle.conf.j2" - dest: "{{ docker_compose.directories.instance | default(lookup('container', application_id, 'directories.instance')) }}/nginx.conf" - mode: "0640" - notify: compose up - -- name: flush docker service - meta: flush_handlers - -- name: Wait for Moodle container to become healthy - block: - - name: Wait until the Moodle container is healthy - ansible.builtin.shell: > - container inspect --format '{% raw %}{{.State.Health.Status}}{% endraw %}' {{ MOODLE_CONTAINER }} - register: health_check - until: health_check.stdout.strip() == "healthy" - retries: 120 - delay: 5 - - rescue: - - name: Show Moodle container status - ansible.builtin.shell: > - container inspect {{ MOODLE_CONTAINER }} - register: moodle_inspect - changed_when: false - failed_when: false - - - name: Show Moodle container logs - ansible.builtin.shell: > - container logs --tail=200 {{ MOODLE_CONTAINER }} - register: moodle_logs - changed_when: false - failed_when: false - - - name: Fail with diagnostic output - ansible.builtin.fail: - msg: | - โŒ Moodle container did not become healthy in time. - - --- container inspect --- - {{ moodle_inspect.stdout | default('no output') }} - - --- container logs (last 200 lines) --- - {{ moodle_logs.stdout | default('no output') }} - -- name: "Run Moodle install (idempotent)" - command: > - container exec --user {{ MOODLE_RUNTIME_USER }} {{ MOODLE_CONTAINER }} - php {{ MOODLE_CLI_DIR }}/install_database.php - --agree-license - --adminuser={{ lookup('config', application_id, 'users.administrator.username') }} - --adminpass={{ lookup('config', application_id, 'credentials.user_password') | quote }} - --adminemail={{ lookup('config', application_id, 'users.administrator.email') }} - --fullname="{{ MOODLE_SITE_TITLE }}" - --shortname=Academy - register: moodle_install - changed_when: "'Moodle has been installed' in moodle_install.stdout" - failed_when: - - moodle_install.rc != 0 - - "'Moodle is already installed' not in moodle_install.stderr" - - "'already installed' not in moodle_install.stdout" - - "'Database tables already present' not in moodle_install.stderr" - no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" - -- name: "Include ownership settings tasks for moodle" - include_tasks: 02_ownership.yml - -- name: "Configure OIDC login for Moodle (variant 0 only)" - include_tasks: 03_oidc.yml - when: MOODLE_OIDC_ENABLED | bool - -- name: "Configure auth_ldap (sync-only in variant 0, primary login in variant 1)" - include_tasks: 04_ldap_sync.yml - when: MOODLE_LDAP_ENABLED | bool + application_id: "web-app-moodle" diff --git a/roles/web-app-navigator/meta/services.yml b/roles/web-app-navigator/meta/services.yml index f44e4773a8..0a3f42f4dc 100644 --- a/roles/web-app-navigator/meta/services.yml +++ b/roles/web-app-navigator/meta/services.yml @@ -11,15 +11,19 @@ sso: enabled: false shared: false dashboard: + bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" matomo: + bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" shared: "{{ 'web-app-matomo' in group_names }}" css: + bond: 1 enabled: "{{ 'web-svc-css' in group_names }}" shared: "{{ 'web-svc-css' in group_names }}" prometheus: + bond: 1 enabled: "{{ 'web-app-prometheus' in group_names }}" shared: "{{ 'web-app-prometheus' in group_names }}" navigator: diff --git a/roles/web-app-nextcloud/README.md b/roles/web-app-nextcloud/README.md index 01aa55638b..dbecbe9cad 100644 --- a/roles/web-app-nextcloud/README.md +++ b/roles/web-app-nextcloud/README.md @@ -17,9 +17,28 @@ This role provisions a complete Nextcloud deployment using Docker Compose. It au - **Integrated Backup & Recovery:** Provides built-in support for backup and restoration operations to safeguard your data. - **Extensible Plugin Framework:** Easily manage and configure hundreds of Nextcloud plugins using the OCC command line tool. +## Addons + +The config-bearing Nextcloud apps are declared in [`meta/addons/`](./meta/addons/) under the unified addon contract (requirement 026). +Each declaration carries its full `occ config:app:set` payload under `config:`. +The enable-only appstore apps stay under `nextcloud.plugins` in [`meta/services.yml`](./meta/services.yml). + +| Addon | Mechanism | Default state | Bridges | +|-------|-----------|---------------|---------| +| `sociallogin` | `plugin` | enabled when the SSO OIDC plugin selector picks it | `sso` โ†’ `web-app-keycloak` | +| `user_ldap` | `plugin` | enabled with the `ldap` service | `ldap` โ†’ `svc-db-openldap` | +| `bbb` | `plugin` | enabled with the `bigbluebutton` partner | `bigbluebutton` โ†’ `web-app-bigbluebutton` | +| `onlyoffice` | `plugin` | enabled with the `onlyoffice` partner | `onlyoffice` โ†’ `web-svc-onlyoffice` | +| `richdocuments` | `plugin` | enabled with the `collabora` partner | `collabora` โ†’ `web-svc-collabora` | +| `spreed` | `plugin` | enabled with the `talk` service | `talk`, `coturn` | +| `whiteboard` | `plugin` | `required` (always installed) | none (self-hosted backend) | +| `xwiki` | `plugin` | enabled with the `xwiki` partner | `xwiki` โ†’ `web-app-xwiki` | + +The SSO (`sociallogin`) and LDAP (`user_ldap`) login surfaces are covered by the OIDC/LDAP Playwright specs (requirements 017/018). + ## Documentation -A detailled documentation for the use and administration of Nextcloud on Infinito.Nexus you will find [here](docs/README.md) +A detailed documentation for the use and administration of Nextcloud on Infinito.Nexus you will find [here](docs/README.md). ## Further Resources diff --git a/roles/web-app-nextcloud/files/gitlab/provision_oauth_app.rb b/roles/web-app-nextcloud/files/gitlab/provision_oauth_app.rb new file mode 100644 index 0000000000..697fadc293 --- /dev/null +++ b/roles/web-app-nextcloud/files/gitlab/provision_oauth_app.rb @@ -0,0 +1,28 @@ +org = (Organizations::Organization.default_organization rescue nil) || (Organizations::Organization.first rescue nil) +raise "GitLab: no organization available to own the Nextcloud OAuth application" unless org +::Current.organization = org if defined?(::Current) && ::Current.respond_to?(:organization=) + +base = ENV.fetch("NC_BASE_URL").sub(%r{/\z}, "") +path = ENV.fetch("NC_REDIRECT_PATH") +redirect_uris = ["#{base}#{path}", "#{base}/index.php#{path}"].join("\n") + +app = Doorkeeper::Application.find_or_initialize_by(name: "Nextcloud") +was_new = app.new_record? +app.redirect_uri = redirect_uris +app.scopes = "api read_user" if app.respond_to?(:scopes=) +app.confidential = true if app.respond_to?(:confidential=) +app.organization = org if app.respond_to?(:organization=) +app.organization_id = org.id if app.respond_to?(:organization_id=) +app.save! + +secret = nil +if was_new + secret = (app.plaintext_secret rescue nil) +elsif ENV["NC_HAS_SECRET"].to_s.strip.empty? && app.respond_to?(:renew_secret) + app.renew_secret + app.save! + secret = (app.plaintext_secret rescue nil) +end + +puts "CLIENT_ID=#{app.uid}" +puts "CLIENT_SECRET=#{secret}" unless secret.to_s.empty? diff --git a/roles/web-app-nextcloud/files/mattermost/provision_oauth_app.sh b/roles/web-app-nextcloud/files/mattermost/provision_oauth_app.sh new file mode 100644 index 0000000000..d6db4e3c04 --- /dev/null +++ b/roles/web-app-nextcloud/files/mattermost/provision_oauth_app.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Idempotently ensure a Mattermost OAuth 2.0 service-provider application exists +# for the Nextcloud integration_mattermost connector and print its credentials. +# +# Runs INSIDE the Mattermost container (admin login + REST API via localhost). +# Emits exactly two machine-parseable lines on success: +# CLIENT_ID= +# CLIENT_SECRET= +# +# Required env: +# MM_LOCAL_URL internal base url, e.g. http://localhost:8065 +# MM_TOKEN admin API token (preferred; e.g. a mmctl-generated PAT). When +# set, MM_ADMIN_LOGIN/MM_ADMIN_PASS are ignored. Required when the +# admin is an SSO user (password login is disabled). +# MM_ADMIN_LOGIN admin login id (email or username) โ€” password-login fallback +# MM_ADMIN_PASS admin password โ€” password-login fallback +# MM_APP_NAME OAuth app display name (stable identity key) +# NC_BASE_URL Nextcloud base url +# NC_REDIRECT_PATH integration_mattermost oauth-redirect path (leading slash) +# MM_HOMEPAGE Nextcloud base url +set -euo pipefail + +api="${MM_LOCAL_URL%/}/api/v4" + +if [ -n "${MM_TOKEN:-}" ]; then + token="${MM_TOKEN}" +else + token="$( + curl -sS -i -X POST "${api}/users/login" \ + -H 'Content-Type: application/json' \ + -d "$(printf '{"login_id":"%s","password":"%s"}' "${MM_ADMIN_LOGIN}" "${MM_ADMIN_PASS}")" \ + | tr -d '\r' \ + | awk 'tolower($1)=="token:"{print $2; exit}' + )" +fi + +if [ -z "${token}" ]; then + echo "mattermost authentication failed: no token available" >&2 + exit 1 +fi + +auth="Authorization: Bearer ${token}" + +existing="$( + curl -sS -H "${auth}" "${api}/oauth/apps?page=0&per_page=200" \ + | python3 -c 'import json,os,sys +name=os.environ["MM_APP_NAME"] +try: + apps=json.load(sys.stdin) +except Exception: + apps=[] +for a in apps if isinstance(apps,list) else []: + if a.get("name")==name: + print("%s\t%s"%(a.get("id",""),a.get("client_secret",""))) + break +' +)" + +if [ -n "${existing}" ]; then + client_id="${existing%% *}" + client_secret="${existing#* }" +else + created="$( + curl -sS -X POST "${api}/oauth/apps" \ + -H "${auth}" -H 'Content-Type: application/json' \ + -d "$(python3 -c 'import json,os +base=os.environ["NC_BASE_URL"].rstrip("/") +path=os.environ["NC_REDIRECT_PATH"] +print(json.dumps({ + "name": os.environ["MM_APP_NAME"], + "description": "Nextcloud integration_mattermost connector", + "homepage": os.environ["MM_HOMEPAGE"], + "callback_urls": [base+path, base+"/index.php"+path], + "is_trusted": True, +}))')" + )" + client_id="$(printf '%s' "${created}" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("id",""))')" + client_secret="$(printf '%s' "${created}" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("client_secret",""))')" +fi + +if [ -z "${client_id}" ] || [ -z "${client_secret}" ]; then + echo "failed to obtain mattermost oauth app credentials" >&2 + exit 1 +fi + +printf 'CLIENT_ID=%s\n' "${client_id}" +printf 'CLIENT_SECRET=%s\n' "${client_secret}" diff --git a/roles/web-app-nextcloud/files/openproject/ensure_nextcloud_storage.rb b/roles/web-app-nextcloud/files/openproject/ensure_nextcloud_storage.rb new file mode 100644 index 0000000000..1d56aeb050 --- /dev/null +++ b/roles/web-app-nextcloud/files/openproject/ensure_nextcloud_storage.rb @@ -0,0 +1,82 @@ +require "json" + +host = ENV.fetch("NEXTCLOUD_HOST").sub(%r{/+\z}, "") +name = ENV.fetch("STORAGE_NAME") +nc_client_id = ENV["NC_OAUTH_CLIENT_ID"].to_s.strip +nc_client_secret = ENV["NC_OAUTH_CLIENT_SECRET"].to_s.strip +nc_has_op_secret = ENV["NC_HAS_OP_SECRET"].to_s.strip + +admins = User.where(admin: true) +creator = + (admins.respond_to?(:active) ? admins.active.first : nil) || + admins.first || + User.where(login: "admin").first + +raise "No administrator user available to own the Nextcloud storage" if creator.nil? + +storage = Storages::NextcloudStorage.find_by(host: host) || + Storages::NextcloudStorage.find_by(name: name) + +if storage.nil? + storage = Storages::NextcloudStorage.new( + name: name, + host: host, + creator: creator + ) + storage.authentication_method = "two_way_oauth2" if storage.respond_to?(:authentication_method=) + storage.save! +else + storage.update!(host: host) if storage.host != host +end + +application = storage.oauth_application +op_client_secret = nil + +if application && nc_client_id.empty? && nc_has_op_secret.empty? + application.destroy + storage.reload + application = nil +end + +redirect_uri = [ + File.join(host, "apps/integration_openproject/oauth-redirect"), + File.join(host, "index.php/apps/integration_openproject/oauth-redirect") +].join("\n") + +if application.nil? + result = ::OAuth::Applications::CreateService + .new(user: creator) + .call( + name: "#{name} (Nextcloud)", + redirect_uri: redirect_uri, + scopes: "api_v3", + confidential: true, + owner: creator, + integration: storage + ) + application = result.is_a?(ServiceResult) ? result.result : result + op_client_secret = application.plaintext_secret if application.respond_to?(:plaintext_secret) + storage.reload +end + +if nc_client_id.present? && nc_client_secret.present? + client = storage.oauth_client + if client.nil? || client.client_id != nc_client_id + if client + client.destroy + storage.reload + end + ::OAuthClients::CreateService + .new(user: creator) + .call(client_id: nc_client_id, client_secret: nc_client_secret, integration: storage) + storage.reload + end +end + +puts JSON.generate( + storage_id: storage.id, + openproject_client_id: storage.oauth_application&.uid, + openproject_client_secret: op_client_secret, + nc_oauth_client_id: storage.oauth_client&.client_id, + oauth_application_created: !op_client_secret.nil? +) diff --git a/roles/web-app-nextcloud/files/playwright/_shared.js b/roles/web-app-nextcloud/files/playwright/_shared.js index c7ced7940c..70f00439e4 100644 --- a/roles/web-app-nextcloud/files/playwright/_shared.js +++ b/roles/web-app-nextcloud/files/playwright/_shared.js @@ -1,20 +1,7 @@ -// Shared Nextcloud Playwright spec state: env vars, locator helpers, modal -// dismissal, login/logout flow, and the `beforeEach` env-presence guard. -// `playwright.spec.js` wires the lifecycle hook and `require()`s one test -// module per scenario so each test stays atomar. Talk-admin-specific -// helpers and env decoding live in `test-talk-admin-settings.js`. - const { expect } = require("@playwright/test"); const { decodeDotenvQuotedValue, findFirstVisibleCandidate, runAdminFlow, runBiberFlow, runGuestFlow } = require("./personas"); const { isServiceEnabled } = require("./service-gating"); -// --------------------------------------------------------------------------- -// Env decoding -// -// All values originate from the rendered `.env` under the staging dir. -// `docker --env-file` preserves the quotes emitted by `dotenv_quote`, so -// normalize these values before building URLs or typing credentials. -// --------------------------------------------------------------------------- const loginUsername = decodeDotenvQuotedValue(process.env.LOGIN_USERNAME); const loginPassword = decodeDotenvQuotedValue(process.env.LOGIN_PASSWORD); const biberUsername = decodeDotenvQuotedValue(process.env.BIBER_USERNAME); @@ -22,17 +9,11 @@ const biberPassword = decodeDotenvQuotedValue(process.env.BIBER_PASSWORD); const nextcloudDirectLoginPassword = decodeDotenvQuotedValue(process.env.NEXTCLOUD_DIRECT_LOGIN_PASSWORD) || loginPassword; const oidcIssuerUrl = decodeDotenvQuotedValue(process.env.OIDC_ISSUER_URL); const nextcloudBaseUrl = decodeDotenvQuotedValue(process.env.NEXTCLOUD_BASE_URL); +const moodleBaseUrl = decodeDotenvQuotedValue(process.env.MOODLE_BASE_URL); +const peertubeBaseUrl = decodeDotenvQuotedValue(process.env.PEERTUBE_BASE_URL); const nextcloudUsernameFieldPattern = /account name(?: or email)?|username(?: or email)?/i; const nextcloudCredentialSubmitPattern = /^(sign in|log in)$/i; -// Condition variables driving the login flavor. Ansible renders these from the -// role's services.{oidc,ldap}.enabled (meta/services.yml) so the spec never has to -// sniff which login UI shape the deployment exposes: -// - OIDC + LDAP -> "oidc_login" (pulsejet/nextcloud-oidc-login, -// auto_redirect hands straight to Keycloak) -// - OIDC only -> "sociallogin" (nextcloud/sociallogin shows a -// "Log in with Keycloak" entry first) -// - no OIDC -> "native" (no Keycloak handoff; NC credential form) const nextcloudOidcEnabled = isServiceEnabled("sso"); const nextcloudLdapEnabled = isServiceEnabled("ldap"); const nextcloudLoginFlavor = !nextcloudOidcEnabled @@ -41,22 +22,10 @@ const nextcloudLoginFlavor = !nextcloudOidcEnabled ? "oidc_login" : "sociallogin"; -// --------------------------------------------------------------------------- -// Locator helpers -// -// Nextcloud renders different "shell" containers depending on the app (Vue -// vs. legacy) and version. The selectors below match any of them so the -// tests work across NC 28+ without hard-coding one layout. -// --------------------------------------------------------------------------- - function getNextcloudShellCandidates(target) { return [ { kind: "shell", - // #app-content-vue: dashboard and Vue-based apps. - // #app-navigation-vue: Vue sidebar (files etc.). - // #app-content: legacy app container. - // #header-start__appmenu: always present in layout.user.php