From d56258b3263177e0bf74f446b66c4fa58480caeb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 02:06:26 +0000 Subject: [PATCH 001/197] chore(deps-dev): bump ruff from 0.16.6 to 0.16.7 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.16.6 to 0.16.7. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.16.6...0.16.7) --- updated-dependencies: - dependency-name: ruff dependency-version: 0.16.7 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 606ff87fab..6744e6d35d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ deploy = [ "requests", ] dev = [ - "ruff==0.16.6", + "ruff==0.16.7", "import-linter", "pre-commit", "git-maintainer-tools", From f47ea0e5bb785a84c136e105eb71e207774f2782 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 15:28:38 +0200 Subject: [PATCH 002/197] fix(discourse): bootstrap only for what the bootstrap consumes Three notifiers pointed at one handler, and that handler always ran ./launcher rebuild. A rotated database password therefore cost a full image build with bundle install and asset precompile: 961.82s in pass 1 and 13m16s in pass 2 of job 103878400427, on a run that then missed its wall-clock budget by 201s. The launcher draws the line itself. set_template_info reads app.yml on every call and turns env: into -e flags that run_start hands to docker run, while templates:, params: and hooks: are consumed during bootstrap only. An env: change needs a new container, not a new image. run_start takes the docker run branch only when no container exists, which is why the start path still stops and removes it first. The digest task is the discriminator: it records a sha256 over every top-level key except env:, so a credential-only change leaves it untouched and the bootstrap handler is never notified. A hash rather than the content because that section carries the OIDC client secret and the administrator password. The guard against running both paths lives in a fact cleared per role run. A register from inside the included file would have outlived the play and, in a later one, silently skipped the recreate while the config change never reached the container. from_yaml adds no failure mode: the launcher parses the same file with YAML.load, so anything it rejects was already a broken deployment. start without an image is structurally impossible, since a running container implies its image and a missing one notifies the bootstrap. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-discourse/handlers/main.yml | 18 +++++++++++++++++- .../web-app-discourse/handlers/stack_host.yml | 4 ++-- roles/web-app-discourse/tasks/03_launcher.yml | 19 +++++++++++++++++-- roles/web-app-discourse/vars/main.yml | 2 ++ 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/roles/web-app-discourse/handlers/main.yml b/roles/web-app-discourse/handlers/main.yml index 524a11676f..94708c8d57 100644 --- a/roles/web-app-discourse/handlers/main.yml +++ b/roles/web-app-discourse/handlers/main.yml @@ -1,5 +1,21 @@ --- +- name: "๐Ÿ”จ Bootstrap the stack host image" + include_tasks: stack_host.yml + vars: + DISCOURSE_LAUNCHER_ACTION: rebuild + when: IS_STACK_HOST | bool + listen: rebuild-discourse + +- name: "๐Ÿ“ Record that the bootstrap already recreated the container" + ansible.builtin.set_fact: + DISCOURSE_REBUILT: true + listen: rebuild-discourse + - name: "๐Ÿ–ฅ๏ธ Handle stack host" include_tasks: stack_host.yml - when: IS_STACK_HOST | bool + vars: + DISCOURSE_LAUNCHER_ACTION: start + when: + - IS_STACK_HOST | bool + - not DISCOURSE_REBUILT | bool listen: recreate-discourse diff --git a/roles/web-app-discourse/handlers/stack_host.yml b/roles/web-app-discourse/handlers/stack_host.yml index 067ad6ac64..e6b013829d 100644 --- a/roles/web-app-discourse/handlers/stack_host.yml +++ b/roles/web-app-discourse/handlers/stack_host.yml @@ -25,9 +25,9 @@ - DISCOURSE_DATABASE_IS_SHARED | bool - DEPLOYMENT_MODE == 'compose' -- name: ๐Ÿ”จ rebuild discourse +- name: "๐Ÿ”จ {{ DISCOURSE_LAUNCHER_ACTION }} discourse" shell: > - ./launcher rebuild {{ DISCOURSE_CONTAINER }} + ./launcher {{ DISCOURSE_LAUNCHER_ACTION }} {{ DISCOURSE_CONTAINER }} {% if DOCKER_IN_CONTAINER | bool %}--skip-prereqs{% endif %} args: executable: /bin/bash diff --git a/roles/web-app-discourse/tasks/03_launcher.yml b/roles/web-app-discourse/tasks/03_launcher.yml index 6a692add06..63187b289d 100644 --- a/roles/web-app-discourse/tasks/03_launcher.yml +++ b/roles/web-app-discourse/tasks/03_launcher.yml @@ -1,4 +1,10 @@ --- +# Exception: set_fact outlives the play, so the guard the recreate handler +# reads has to be cleared per role run or a stale true silently skips it. +- name: "๐Ÿงน Clear the bootstrap marker of an earlier play" + ansible.builtin.set_fact: + DISCOURSE_REBUILT: false + - name: โฌ‡๏ธ Pull Discourse docker repository git: repo: "{{ DISCOURSE_REPOSITORY_URL }}" @@ -10,7 +16,7 @@ retries: 12 delay: 30 until: discourse_repo_pull is succeeded - notify: recreate-discourse + notify: rebuild-discourse become: true - name: ๐Ÿ”ง set chmod 700 for '{{ DISCOURSE_CONTAINERS_DIR }}' @@ -26,6 +32,15 @@ mode: "0640" notify: recreate-discourse +# Exception: launcher reads env: on every docker run but consumes templates, +# params and hooks only during bootstrap, so only their digest may rebuild. +- name: "๐Ÿ”‘ Record the digest of everything the bootstrap consumes" + stack_host_copy: + content: "{{ lookup('template', 'config.yml.j2') | from_yaml | dict2items | rejectattr('key', 'eq', 'env') | items2dict | to_json | hash('sha256') }}" + dest: "{{ DISCOURSE_BOOTSTRAP_DIGEST_DEST }}" + mode: "0640" + notify: rebuild-discourse + - name: "โœ… Verify that '{{ DISCOURSE_CONTAINER }}' is running" ansible.builtin.command: argv: @@ -40,7 +55,7 @@ register: docker_ps changed_when: docker_ps.stdout.strip() == "" failed_when: docker_ps.rc != 0 - notify: recreate-discourse + notify: rebuild-discourse - name: ๐Ÿ“‚ Load discourse handlers explicitly include_tasks: utils/load/handlers.yml diff --git a/roles/web-app-discourse/vars/main.yml b/roles/web-app-discourse/vars/main.yml index 909fb515cc..f768e247aa 100644 --- a/roles/web-app-discourse/vars/main.yml +++ b/roles/web-app-discourse/vars/main.yml @@ -30,6 +30,8 @@ DISCOURSE_CONTAINERS_DIR: "{{ [ DISCOURSE_REPOSITORY_DIR, 'containers' ] | path_ DISCOURSE_APPLICATION_YML_DEST: "{{ [ DISCOURSE_CONTAINERS_DIR, DISCOURSE_CONTAINER ~ '.yml' ] | path_join }}" +DISCOURSE_BOOTSTRAP_DIGEST_DEST: "{{ [ DISCOURSE_CONTAINERS_DIR, DISCOURSE_CONTAINER ~ '.bootstrap.sha256' ] | path_join }}" + DISCOURSE_LDAP_ENABLED: "{{ lookup('config', application_id, 'services.ldap.enabled') }}" DISCOURSE_LITELLM_ENABLED: "{{ lookup('config', application_id, 'services.litellm.enabled') | bool }}" From e11c9517b294a78f8eed66274f916468576bdcf8 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 16:07:15 +0200 Subject: [PATCH 003/197] fix(discourse): set runtime state at runtime, not during bootstrap All 22 run: hooks in app.yml were rails and rake calls against the database, yet the launcher executes them only while building the image. Two of them carried rotating values, the OIDC client secret and the administrator password, so every rotation forced a full rebuild with bundle install and asset precompile: 961.82s in one pass and 13m16s in the next, on a job that then missed its wall-clock budget by 201s. They now run against the running instance through container exec, following the pattern web-app-openproject already uses: real .rb files under files/ruby, values through ENV.fetch, a readback that prints booleans rather than values, and an assert that fails the deploy when a rotated credential did not arrive. ensure_admin_user.rb mirrors the Y/Y path of admin:create using upstream's own grant_admin!, change_trust_level! and activate; setting the flags by hand had skipped auto_approve_user, the staff welcome message, the notification levels, trust level 1 and the email confirmation. What stays in run: is the CA trust call, which writes into the image and would not survive a recreate. With the rest gone, the bootstrap digest f47ea0e5bb records covers templates, expose, params, volumes, after_code and docker_args, and nothing in that set rotates. 00_core.yml carried the same IS_STACK_HOST condition three times; it now includes 04_stack_host.yml once, which holds the three with their own conditions, and their implementations moved under tasks/stack_host/. The OIDC values are read only when OIDC is enabled. Ansible resolves vars lazily, so referencing DISCOURSE_OIDC_LOGOUT_REDIRECT_URL unconditionally would have run its lookup against web-svc-logout on a deployment that has no SSO, failing where the template did not. test_oidc_setting_order.py follows the settings into apply_settings.rb. The invariant 88218fc740 introduced is unchanged: openid_connect_enabled comes after the discovery document, the client id and the client secret. Four explanatory "# Exception:" comments are removed, two of them from f47ea0e5bb. They described mechanisms rather than naming a trip-wire, which belongs in a commit body. Co-Authored-By: Claude Opus 5 (1M context) --- .../files/ruby/apply_settings.rb | 35 ++++++++ .../files/ruby/ensure_admin_user.rb | 19 ++++ .../files/ruby/verify_settings.rb | 17 ++++ roles/web-app-discourse/tasks/00_core.yml | 14 +-- roles/web-app-discourse/tasks/03_launcher.yml | 4 - .../web-app-discourse/tasks/04_stack_host.yml | 11 +++ .../01_network.yml} | 0 .../tasks/stack_host/02_credentials.yml | 87 +++++++++++++++++++ .../web-app-discourse/templates/config.yml.j2 | 69 --------------- .../test_oidc_setting_order.py | 6 +- 10 files changed, 175 insertions(+), 87 deletions(-) create mode 100644 roles/web-app-discourse/files/ruby/apply_settings.rb create mode 100644 roles/web-app-discourse/files/ruby/ensure_admin_user.rb create mode 100644 roles/web-app-discourse/files/ruby/verify_settings.rb create mode 100644 roles/web-app-discourse/tasks/04_stack_host.yml rename roles/web-app-discourse/tasks/{04_network.yml => stack_host/01_network.yml} (100%) create mode 100644 roles/web-app-discourse/tasks/stack_host/02_credentials.yml diff --git a/roles/web-app-discourse/files/ruby/apply_settings.rb b/roles/web-app-discourse/files/ruby/apply_settings.rb new file mode 100644 index 0000000000..6c212121f5 --- /dev/null +++ b/roles/web-app-discourse/files/ruby/apply_settings.rb @@ -0,0 +1,35 @@ +# nocheck: mirrored-unit-test - runs in the Discourse console against SiteSetting and +# Discourse.cache; both only exist once the application has booted + +require "json" + +SiteSetting.force_https = ENV.fetch("DISCOURSE_FORCE_HTTPS") == "true" +SiteSetting.enable_local_logins = ENV.fetch("DISCOURSE_LOCAL_LOGINS") == "true" + +JSON.parse(ENV.fetch("DISCOURSE_LDAP_SITE_SETTINGS")).each do |key, value| + SiteSetting.public_send("#{key}=", value) +end + +if ENV.fetch("DISCOURSE_OIDC_ENABLED") == "true" + SiteSetting.openid_connect_verbose_logging = ENV.fetch("DISCOURSE_OIDC_VERBOSE") == "true" + SiteSetting.enable_passkeys = false + SiteSetting.username_change_period = 0 + + SiteSetting.openid_connect_discovery_document = ENV.fetch("DISCOURSE_OIDC_DISCOVERY_DOCUMENT") + SiteSetting.openid_connect_client_id = ENV.fetch("DISCOURSE_OIDC_CLIENT_ID") + SiteSetting.openid_connect_client_secret = ENV.fetch("DISCOURSE_OIDC_CLIENT_SECRET") + SiteSetting.openid_connect_enabled = true + SiteSetting.openid_connect_rp_initiated_logout_redirect = ENV.fetch("DISCOURSE_OIDC_LOGOUT_REDIRECT") + SiteSetting.openid_connect_allow_association_change = false + SiteSetting.openid_connect_rp_initiated_logout = true + + url = SiteSetting.openid_connect_discovery_document.to_s.strip + host = url.sub(%r{\Ahttps?://}i, "").split("/").first.to_s.strip.split(":").first.to_s.strip + list = SiteSetting.allowed_internal_hosts.to_s.split("|").map(&:strip).reject(&:empty?) + if !host.empty? && host =~ /\A[a-z0-9.-]+\z/i && !list.include?(host) + list << host + SiteSetting.allowed_internal_hosts = list.join("|") + end + + Discourse.cache.delete("openid-connect-discovery-#{url}") +end diff --git a/roles/web-app-discourse/files/ruby/ensure_admin_user.rb b/roles/web-app-discourse/files/ruby/ensure_admin_user.rb new file mode 100644 index 0000000000..553ed9e5c8 --- /dev/null +++ b/roles/web-app-discourse/files/ruby/ensure_admin_user.rb @@ -0,0 +1,19 @@ +# nocheck: mirrored-unit-test - runs in the Discourse console against User and its +# password validators; the models only exist once the application has booted + +email = ENV.fetch("DISCOURSE_ADMIN_EMAIL") +username = ENV.fetch("DISCOURSE_ADMIN_USERNAME") +password = ENV.fetch("DISCOURSE_ADMIN_PASSWORD") + +user = User.find_by_email(email) || User.new(email: email) +user.username = username +user.password = password +user.save! + +user.active = true +user.save! + +user.grant_admin! +user.change_trust_level!(1) if user.trust_level < 1 +user.email_tokens.update_all(confirmed: true) +user.activate diff --git a/roles/web-app-discourse/files/ruby/verify_settings.rb b/roles/web-app-discourse/files/ruby/verify_settings.rb new file mode 100644 index 0000000000..202ab7300a --- /dev/null +++ b/roles/web-app-discourse/files/ruby/verify_settings.rb @@ -0,0 +1,17 @@ +# nocheck: mirrored-unit-test - reads SiteSetting and User out of the booted +# application, neither of which exists outside the Discourse console + +email = ENV.fetch("DISCOURSE_ADMIN_EMAIL") +password = ENV.fetch("DISCOURSE_ADMIN_PASSWORD") + +user = User.find_by_email(email) +puts "administrator_present=#{!user.nil?}" +puts "administrator_admin=#{user&.admin == true}" +puts "administrator_active=#{user&.active == true}" +puts "administrator_password_matches=#{user&.confirm_password?(password) == true}" + +if ENV.fetch("DISCOURSE_OIDC_ENABLED") == "true" + puts "oidc_secret_matches=#{SiteSetting.openid_connect_client_secret == ENV.fetch('DISCOURSE_OIDC_CLIENT_SECRET')}" + puts "oidc_enabled=#{SiteSetting.openid_connect_enabled == true}" + puts "oidc_host_allowed=#{SiteSetting.allowed_internal_hosts.to_s.split('|').any? { |h| SiteSetting.openid_connect_discovery_document.to_s.include?(h.strip) }}" +end diff --git a/roles/web-app-discourse/tasks/00_core.yml b/roles/web-app-discourse/tasks/00_core.yml index ffa18be27b..00f0e35171 100644 --- a/roles/web-app-discourse/tasks/00_core.yml +++ b/roles/web-app-discourse/tasks/00_core.yml @@ -9,14 +9,6 @@ - name: "๐Ÿ”ง Setup '{{ application_id }}' docker" include_tasks: 02_docker.yml -- name: "๐Ÿ”ง Setup '{{ application_id }}' network" - include_tasks: 04_network.yml - when: - - IS_STACK_HOST | bool - - lookup('database', application_id, 'shared') - -- name: "๐Ÿค– Bind the native AI surface to the platform gateway" - include_tasks: addons/discourse-ai.yml - when: - - IS_STACK_HOST | bool - - DISCOURSE_LITELLM_ENABLED | bool +- name: "๐Ÿ–ฅ๏ธ Setup what only the stack host owns" + include_tasks: 04_stack_host.yml + when: IS_STACK_HOST | bool diff --git a/roles/web-app-discourse/tasks/03_launcher.yml b/roles/web-app-discourse/tasks/03_launcher.yml index 63187b289d..f816df8bcc 100644 --- a/roles/web-app-discourse/tasks/03_launcher.yml +++ b/roles/web-app-discourse/tasks/03_launcher.yml @@ -1,6 +1,4 @@ --- -# Exception: set_fact outlives the play, so the guard the recreate handler -# reads has to be cleared per role run or a stale true silently skips it. - name: "๐Ÿงน Clear the bootstrap marker of an earlier play" ansible.builtin.set_fact: DISCOURSE_REBUILT: false @@ -32,8 +30,6 @@ mode: "0640" notify: recreate-discourse -# Exception: launcher reads env: on every docker run but consumes templates, -# params and hooks only during bootstrap, so only their digest may rebuild. - name: "๐Ÿ”‘ Record the digest of everything the bootstrap consumes" stack_host_copy: content: "{{ lookup('template', 'config.yml.j2') | from_yaml | dict2items | rejectattr('key', 'eq', 'env') | items2dict | to_json | hash('sha256') }}" diff --git a/roles/web-app-discourse/tasks/04_stack_host.yml b/roles/web-app-discourse/tasks/04_stack_host.yml new file mode 100644 index 0000000000..bbf7175342 --- /dev/null +++ b/roles/web-app-discourse/tasks/04_stack_host.yml @@ -0,0 +1,11 @@ +--- +- name: "๐Ÿ”ง Setup '{{ application_id }}' network" + include_tasks: stack_host/01_network.yml + when: lookup('database', application_id, 'shared') + +- name: "๐Ÿ”‘ Apply the credentials the bootstrap must not carry" + include_tasks: stack_host/02_credentials.yml + +- name: "๐Ÿค– Bind the native AI surface to the platform gateway" + include_tasks: addons/discourse-ai.yml + when: DISCOURSE_LITELLM_ENABLED | bool diff --git a/roles/web-app-discourse/tasks/04_network.yml b/roles/web-app-discourse/tasks/stack_host/01_network.yml similarity index 100% rename from roles/web-app-discourse/tasks/04_network.yml rename to roles/web-app-discourse/tasks/stack_host/01_network.yml diff --git a/roles/web-app-discourse/tasks/stack_host/02_credentials.yml b/roles/web-app-discourse/tasks/stack_host/02_credentials.yml new file mode 100644 index 0000000000..5e6efbb6c2 --- /dev/null +++ b/roles/web-app-discourse/tasks/stack_host/02_credentials.yml @@ -0,0 +1,87 @@ +--- +- name: "โš™๏ธ Apply the site settings to the running instance" + ansible.builtin.command: + argv: + - container + - exec + - -i + - -e + - "DISCOURSE_FORCE_HTTPS={{ DISCOURSE_TLS_ENABLED | bool | string | lower }}" + - -e + - "DISCOURSE_LOCAL_LOGINS={{ DISCOURSE_LOCAL_LOGIN_ENABLED | bool | string | lower }}" + - -e + - "DISCOURSE_LDAP_SITE_SETTINGS={{ (DISCOURSE_PLUGINS['discourse-ldap-auth'].config.site_settings if DISCOURSE_LDAP_ENABLED | bool else {}) | to_json }}" + - -e + - "DISCOURSE_OIDC_ENABLED={{ DISCOURSE_OIDC_ENABLED | bool | string | lower }}" + - -e + - "DISCOURSE_OIDC_VERBOSE={{ MODE_DEBUG | bool | string | lower }}" + - -e + - "DISCOURSE_OIDC_DISCOVERY_DOCUMENT={{ OIDC.CLIENT.DISCOVERY_DOCUMENT if DISCOURSE_OIDC_ENABLED | bool else '' }}" + - -e + - "DISCOURSE_OIDC_CLIENT_ID={{ OIDC.CLIENT.ID if DISCOURSE_OIDC_ENABLED | bool else '' }}" + - -e + - "DISCOURSE_OIDC_CLIENT_SECRET={{ OIDC.CLIENT.SECRET if DISCOURSE_OIDC_ENABLED | bool else '' }}" + - -e + - "DISCOURSE_OIDC_LOGOUT_REDIRECT={{ DISCOURSE_OIDC_LOGOUT_REDIRECT_URL if DISCOURSE_OIDC_ENABLED | bool else '' }}" + - "{{ DISCOURSE_CONTAINER }}" # nocheck: container-exec-resolver the launcher names the container itself, so the bare name resolves in both modes + - rails + - r + - "{{ lookup('file', 'ruby/apply_settings.rb') }}" + changed_when: false + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "๐Ÿ”‘ Ensure the administrator account carries the current password" + ansible.builtin.command: + argv: + - container + - exec + - -i + - -e + - "DISCOURSE_ADMIN_EMAIL={{ lookup('users', 'administrator').email }}" + - -e + - "DISCOURSE_ADMIN_USERNAME={{ lookup('users', 'administrator').username }}" + - -e + - "DISCOURSE_ADMIN_PASSWORD={{ lookup('users', 'administrator').password }}" + - "{{ DISCOURSE_CONTAINER }}" # nocheck: container-exec-resolver the launcher names the container itself, so the bare name resolves in both modes + - rails + - r + - "{{ lookup('file', 'ruby/ensure_admin_user.rb') }}" + changed_when: false + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "๐Ÿ”Ž Read the applied settings back out of the running instance" + ansible.builtin.command: + argv: + - container + - exec + - -i + - -e + - "DISCOURSE_ADMIN_EMAIL={{ lookup('users', 'administrator').email }}" + - -e + - "DISCOURSE_ADMIN_PASSWORD={{ lookup('users', 'administrator').password }}" + - -e + - "DISCOURSE_OIDC_ENABLED={{ DISCOURSE_OIDC_ENABLED | bool | string | lower }}" + - -e + - "DISCOURSE_OIDC_CLIENT_SECRET={{ OIDC.CLIENT.SECRET if DISCOURSE_OIDC_ENABLED | bool else '' }}" + - "{{ DISCOURSE_CONTAINER }}" # nocheck: container-exec-resolver the launcher names the container itself, so the bare name resolves in both modes + - rails + - r + - "{{ lookup('file', 'ruby/verify_settings.rb') }}" + register: discourse_settings_readback + changed_when: false + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "๐Ÿ›‘ Refuse an instance the rotated credentials did not reach" + ansible.builtin.assert: + that: + - "'administrator_present=true' in discourse_settings_readback.stdout_lines" + - "'administrator_admin=true' in discourse_settings_readback.stdout_lines" + - "'administrator_active=true' in discourse_settings_readback.stdout_lines" + - "'administrator_password_matches=true' in discourse_settings_readback.stdout_lines" + - "not DISCOURSE_OIDC_ENABLED | bool or 'oidc_secret_matches=true' in discourse_settings_readback.stdout_lines" + - "not DISCOURSE_OIDC_ENABLED | bool or 'oidc_enabled=true' in discourse_settings_readback.stdout_lines" + - "not DISCOURSE_OIDC_ENABLED | bool or 'oidc_host_allowed=true' in discourse_settings_readback.stdout_lines" + fail_msg: >- + The administrator account, the OIDC client secret or the internal-host allowlist + of the running Discourse does not match the inventory, so a rotated credential + or a site setting never reached it. diff --git a/roles/web-app-discourse/templates/config.yml.j2 b/roles/web-app-discourse/templates/config.yml.j2 index 1cca0ac788..e775e252a8 100644 --- a/roles/web-app-discourse/templates/config.yml.j2 +++ b/roles/web-app-discourse/templates/config.yml.j2 @@ -149,78 +149,9 @@ hooks: ## Any custom commands to run after building run: - exec: echo "Beginning of custom commands" - - exec: rails r "SiteSetting.force_https = {{ DISCOURSE_TLS_ENABLED | bool | string | lower }}" {% if DISCOURSE_TLS_SELFSIGNED_ENABLED | bool %} - exec: {{ BIN_CA_TRUST }} {% endif %} - ## If you want to set the 'From' email address for your first registration, uncomment and change: - ## After getting the first signup email, re-comment the line. It only needs to run once. - #- exec: rails r "SiteSetting.notification_email='info@unconfigured.discourse.org'" - - exec: rails r "SiteSetting.enable_local_logins = {{ DISCOURSE_LOCAL_LOGIN_ENABLED | bool | string | lower }}" -{% if DISCOURSE_OIDC_ENABLED | bool %} - # Deactivate Default Login - - exec: rails r "SiteSetting.openid_connect_verbose_logging = {{ MODE_DEBUG | bool | string | lower }}" - - exec: rails r "SiteSetting.enable_passkeys = false" # https://meta.discourse.org/t/passwordless-login-using-passkeys/285589 - - - exec: rails r "SiteSetting.username_change_period = 0" # Deactivate changing of username - - # OIDC Activation - - exec: rails r "SiteSetting.openid_connect_discovery_document = '{{OIDC.CLIENT.DISCOVERY_DOCUMENT}}'" - - exec: rails r "SiteSetting.openid_connect_client_id = '{{ OIDC.CLIENT.ID }}'" - - exec: rails r "SiteSetting.openid_connect_client_secret = '{{ OIDC.CLIENT.SECRET }}'" - - exec: rails r "SiteSetting.openid_connect_enabled = true" - - exec: rails r "SiteSetting.openid_connect_rp_initiated_logout_redirect = '{{ DISCOURSE_OIDC_LOGOUT_REDIRECT_URL }}'" - - exec: rails r "SiteSetting.openid_connect_allow_association_change = false" - - exec: rails r "SiteSetting.openid_connect_rp_initiated_logout = true" - - # ---- FinalDestination allowlist (IMPORTANT) ---- - # Discourse blocks requests to internal/private IPs unless the host is allowlisted. - # Extract host from discovery document URL and append to allowed_internal_hosts. - - exec: > - rails r "url=SiteSetting.openid_connect_discovery_document.to_s.strip; - host=url.sub(%r{\Ahttps?://}i,'').split('/').first.to_s.strip; - host=host.split(':').first.to_s.strip; - list=SiteSetting.allowed_internal_hosts.to_s.split('|').map{|x| x.strip}.reject{|x| x.empty?}; - if !host.empty? && host =~ /\A[a-z0-9.-]+\z/i && !list.include?(host) - list << host; - SiteSetting.allowed_internal_hosts = list.join('|'); - end; - puts \"allowed_internal_hosts=#{SiteSetting.allowed_internal_hosts}\";" - - # Clear OIDC discovery cache so the next login attempt refetches with the new allowlist - - exec: > - rails r "url=SiteSetting.openid_connect_discovery_document.to_s.strip; - key=\"openid-connect-discovery-#{url}\"; - Discourse.cache.delete(key); - puts \"deleted_cache_key=#{key}\";" - -{% endif %} - - # Activate Administrator User (always yes, non-interactive) - - exec: | - cat <<'EOF' | rake admin:create - {{ lookup('users', 'administrator').email }} - {{ lookup('users', 'administrator').password }} - {{ lookup('users', 'administrator').password }} - Y - Y - EOF - - - exec: > - rails r "User.find_by_email('{{ lookup('users', 'administrator').email }}')&.update!(username: '{{ lookup('users', 'administrator').username }}')" - -{% if DISCOURSE_LDAP_ENABLED | bool %} -{% for setting_key, setting_value in DISCOURSE_PLUGINS['discourse-ldap-auth'].config.site_settings.items() %} -{% if setting_value is boolean %} - - exec: rails r "SiteSetting.{{ setting_key }} = {{ setting_value | string | lower }}" -{% elif setting_value is string %} - - exec: rails r "SiteSetting.{{ setting_key }} = '{{ setting_value }}'" -{% else %} - - exec: rails r "SiteSetting.{{ setting_key }} = {{ setting_value }}" -{% endif %} -{% endfor %} -{% endif %} - - exec: echo "End of custom commands" docker_args: diff --git a/tests/unit/python/roles/web-app-discourse/test_oidc_setting_order.py b/tests/unit/python/roles/web-app-discourse/test_oidc_setting_order.py index d5eb180e00..e619fb2bb4 100644 --- a/tests/unit/python/roles/web-app-discourse/test_oidc_setting_order.py +++ b/tests/unit/python/roles/web-app-discourse/test_oidc_setting_order.py @@ -4,7 +4,7 @@ import unittest from utils.cache.files import PROJECT_ROOT, read_text -CONFIG = PROJECT_ROOT / "roles/web-app-discourse/templates/config.yml.j2" +SETTINGS = PROJECT_ROOT / "roles/web-app-discourse/files/ruby/apply_settings.rb" PREREQUISITES = ( "openid_connect_discovery_document", "openid_connect_client_id", @@ -16,13 +16,13 @@ def _line_of(lines: list[str], setting: str) -> int: return next( index for index, line in enumerate(lines) - if f'rails r "SiteSetting.{setting} = ' in line + if f"SiteSetting.{setting} = " in line ) class TestDiscourseOidcSettingOrder(unittest.TestCase): def test_oidc_is_enabled_only_after_its_prerequisites_are_set(self) -> None: - lines = read_text(str(CONFIG)).splitlines() + lines = read_text(str(SETTINGS)).splitlines() enable = _line_of(lines, "openid_connect_enabled") for setting in PREREQUISITES: with self.subTest(setting=setting): From 4c2e480d5980afe78ca449faa9785e2c5b8c2fe6 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 16:41:35 +0200 Subject: [PATCH 004/197] feat(discourse): pin the application version where the updater can see it params.version sat commented out in app.yml, so Discourse silently inherited tests-passed and every bootstrap could pull a different application revision with nothing recording which. Declaring it as services.discourse.version would have been invisible to both updaters: utils/update/docker.py reaches version: only through iter_role_images, which skips any service without an image: (discovery.py, "if not image or not version: continue"), and Discourse is built by the launcher and has none. A nested application entry carrying repository: and ref: lands in utils/update/repository.py instead, whose walk_repo_ref_pairs descends the whole tree; the collector now reports discourse/application | discourse.git | v2026.8.0 so git ls-remote --tags keeps the pin current rather than freezing it, and the external URL test sees the repository through iter_non_ignored_files. This is the shape 7b15c61dfc established when it migrated version/branch keys to ref. v2026.8.0 is the newest stable tag: v2026.9.0 answers 404, only v2026.9.0-latest exists. The auto-bump will not drift onto those pointers, version_flavor("v2026.8.0") is empty and latest_semver with that flavor picks the plain tag back. Pinning the core while the plugins clone from their default branch is safe because upstream solves it: docker_manager, discourse-akismet and discourse-activity-pub each ship a .discourse-compatibility map, and the bootstrap checks out the plugin commit matching an older core. The newest threshold in those maps is 2026.4.0-latest, below this pin, so the default branch is the intended pairing. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-discourse/meta/services.yml | 3 +++ roles/web-app-discourse/templates/config.yml.j2 | 2 +- roles/web-app-discourse/vars/main.yml | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/web-app-discourse/meta/services.yml b/roles/web-app-discourse/meta/services.yml index c3694d4013..1ce47ba1e0 100644 --- a/roles/web-app-discourse/meta/services.yml +++ b/roles/web-app-discourse/meta/services.yml @@ -69,6 +69,9 @@ discourse: ref: main min_storage: 15GB repository: https://github.com/discourse/discourse_docker.git + application: + repository: https://github.com/discourse/discourse.git + ref: v2026.8.0 plugins: docker_manager: enabled: true diff --git a/roles/web-app-discourse/templates/config.yml.j2 b/roles/web-app-discourse/templates/config.yml.j2 index e775e252a8..91036b29c8 100644 --- a/roles/web-app-discourse/templates/config.yml.j2 +++ b/roles/web-app-discourse/templates/config.yml.j2 @@ -29,7 +29,7 @@ params: #db_work_mem: "40MB" ## Which Git revision should this container use? (default: tests-passed) - #version: tests-passed + version: {{ DISCOURSE_VERSION }} env: LC_ALL: "{{ DISCOURSE_LOCAL }}" diff --git a/roles/web-app-discourse/vars/main.yml b/roles/web-app-discourse/vars/main.yml index f768e247aa..e56af0c86f 100644 --- a/roles/web-app-discourse/vars/main.yml +++ b/roles/web-app-discourse/vars/main.yml @@ -17,6 +17,7 @@ DISCOURSE_REDIS_PORT: "{{ lookup('engine', 'redis', application_id, 'port') }}" DISCOURSE_REDIS_PASSWORD: "{{ lookup('engine', 'redis', application_id, 'password') }}" DISCOURSE_REPOSITORY_URL: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.repository') }}" DISCOURSE_REF: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.ref') }}" +DISCOURSE_VERSION: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.application.ref') }}" DISCOURSE_LDAP_AUTH_PLUGIN_REPOSITORY: "{{ lookup('config', application_id, 'addons.discourse-ldap-auth.config.repository') }}" DISCOURSE_LDAP_AUTH_PLUGIN_REF: "{{ lookup('config', application_id, 'addons.discourse-ldap-auth.config.ref') }}" From 84a4b04f05b7d6f6defaaec90c2ab3147b3258d7 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 16:43:17 +0200 Subject: [PATCH 005/197] refactor(discourse): make meta/addons the one place a plugin is pinned The pinning sat in two places and covered one plugin out of four. discourse-ldap-auth carried repository and ref both in the services.yml plugins map and in its addon file, the duplicate silenced by a "# nocheck: repository-declared". The three official plugins carried neither, so the template hardcoded https://github.com/discourse/.git and cloned whatever the default branch held, while only the LDAP one got a --branch through a special-cased block beside the loop. The addon layer was already the real source: DISCOURSE_PLUGINS reads lookup('config', application_id, 'addons'), and the services.yml plugins block survived only in nocheck comments pointing at it. It is dropped. Every cloned addon now declares its own repository and ref plus the update contract, so the template reads the declaration instead of inventing it and one loop covers all four. Checked against the assembled addon map: the four with source != bundled all carry repository and ref; discourse-ai carries neither and is bundled, which the loop condition already excludes. Pinning these while 4c2e480d5 pins the core is what upstream expects. docker_manager, discourse-akismet and discourse-activity-pub each ship a .discourse-compatibility map from core versions to plugin commits, and the bootstrap falls back to the matching commit for an older core. Their newest threshold is 2026.4.0-latest, below the pinned core, so the declared branch is the intended pairing rather than a drift. Co-Authored-By: Claude Opus 5 (1M context) --- .../meta/addons/discourse-activity-pub.yml | 9 ++++++++- .../meta/addons/discourse-akismet.yml | 9 ++++++++- .../web-app-discourse/meta/addons/docker_manager.yml | 9 ++++++++- roles/web-app-discourse/meta/services.yml | 11 ----------- roles/web-app-discourse/templates/config.yml.j2 | 6 +----- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/roles/web-app-discourse/meta/addons/discourse-activity-pub.yml b/roles/web-app-discourse/meta/addons/discourse-activity-pub.yml index c7dca961bf..05bcb24e16 100644 --- a/roles/web-app-discourse/meta/addons/discourse-activity-pub.yml +++ b/roles/web-app-discourse/meta/addons/discourse-activity-pub.yml @@ -1,6 +1,13 @@ --- -# nocheck: addons-variant-pinning: compiled into the image from services.discourse.plugins, so an addon-level disable saves no deploy time and only silences the spec +# nocheck: addons-variant-pinning: compiled into the image from the addon's ref, so an addon-level disable saves no deploy time and only silences the spec enabled: true required: false mechanism: plugin source: upstream +update: + monitored: true + catalog: discourse-meta + upstream_id: discourse-activity-pub +config: + repository: https://github.com/discourse/discourse-activity-pub.git + ref: main diff --git a/roles/web-app-discourse/meta/addons/discourse-akismet.yml b/roles/web-app-discourse/meta/addons/discourse-akismet.yml index c7dca961bf..770e3fa21d 100644 --- a/roles/web-app-discourse/meta/addons/discourse-akismet.yml +++ b/roles/web-app-discourse/meta/addons/discourse-akismet.yml @@ -1,6 +1,13 @@ --- -# nocheck: addons-variant-pinning: compiled into the image from services.discourse.plugins, so an addon-level disable saves no deploy time and only silences the spec +# nocheck: addons-variant-pinning: compiled into the image from the addon's ref, so an addon-level disable saves no deploy time and only silences the spec enabled: true required: false mechanism: plugin source: upstream +update: + monitored: true + catalog: discourse-meta + upstream_id: discourse-akismet +config: + repository: https://github.com/discourse/discourse-akismet.git + ref: main diff --git a/roles/web-app-discourse/meta/addons/docker_manager.yml b/roles/web-app-discourse/meta/addons/docker_manager.yml index c7dca961bf..8af7b2512d 100644 --- a/roles/web-app-discourse/meta/addons/docker_manager.yml +++ b/roles/web-app-discourse/meta/addons/docker_manager.yml @@ -1,6 +1,13 @@ --- -# nocheck: addons-variant-pinning: compiled into the image from services.discourse.plugins, so an addon-level disable saves no deploy time and only silences the spec +# nocheck: addons-variant-pinning: compiled into the image from the addon's ref, so an addon-level disable saves no deploy time and only silences the spec enabled: true required: false mechanism: plugin source: upstream +update: + monitored: true + catalog: discourse-meta + upstream_id: docker_manager +config: + repository: https://github.com/discourse/docker_manager.git + ref: main diff --git a/roles/web-app-discourse/meta/services.yml b/roles/web-app-discourse/meta/services.yml index 1ce47ba1e0..162066122a 100644 --- a/roles/web-app-discourse/meta/services.yml +++ b/roles/web-app-discourse/meta/services.yml @@ -72,17 +72,6 @@ discourse: application: repository: https://github.com/discourse/discourse.git ref: v2026.8.0 - plugins: - docker_manager: - enabled: true - discourse-activity-pub: - enabled: true - discourse-akismet: - enabled: true - discourse-ldap-auth: - enabled: "{{ lookup('config', 'web-app-discourse', 'services.ldap.enabled') | bool }}" - repository: https://github.com/jonmbake/discourse-ldap-auth.git - ref: master ports: local: http: 8020 diff --git a/roles/web-app-discourse/templates/config.yml.j2 b/roles/web-app-discourse/templates/config.yml.j2 index 91036b29c8..1312e2281d 100644 --- a/roles/web-app-discourse/templates/config.yml.j2 +++ b/roles/web-app-discourse/templates/config.yml.j2 @@ -138,14 +138,10 @@ hooks: cmd: {% for plugin_name, plugin_config in DISCOURSE_PLUGINS.items() %} {% if plugin_config.enabled and plugin_config.source != 'bundled' %} - - git clone --depth=1 https://github.com/discourse/{{ plugin_name }}.git + - git clone --depth=1 --branch {{ plugin_config.config.ref }} {{ plugin_config.config.repository }} {% endif %} {% endfor %} -{% if DISCOURSE_LDAP_ENABLED | bool %} - - git clone --depth=1 --branch {{ DISCOURSE_LDAP_AUTH_PLUGIN_REF }} {{ DISCOURSE_LDAP_AUTH_PLUGIN_REPOSITORY }} -{% endif %} - ## Any custom commands to run after building run: - exec: echo "Beginning of custom commands" From e4141cb82629dc9d4a35af2c4f39af6295436f45 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 16:44:57 +0200 Subject: [PATCH 006/197] feat(update): let the ref updater see the addon declarations collect_entries walked roles/*/meta/services.yml only, so a repository and ref declared in an addon file was invisible to it. That mattered once 84a4b04f0 made meta/addons the one place a Discourse plugin is pinned: the pins would have left the updater's view entirely. The other updater cannot stand in, utils/update/docker.py reaches version: through iter_role_images and skips any service without an image:. e317f08ac9 built the addon contract with its own catalog-driven SPOT in utils/update/addons.py and left repository.py untouched without saying why, so this widens the ref path rather than replacing the catalog one; an addon can be maintained by either. The per-file work moves into _entries_of_file so services.yml and every meta/addons/*.yml share it. Addon entries carry an ("addons", ) prefix on their entity path, which keeps them distinguishable from a service entry holding the same ref. The line index, the semver filter and the per-file "# nocheck: repository-version" suppression apply unchanged. Two tests cover it: an addon ref is collected beside a service ref with its own path and config_path, and a suppressed addon ref yields nothing. Against this repository the extension currently collects nothing, because every addon ref is a branch and is_semver rejects those; the tests are what prove the walk rather than the absence of output. Co-Authored-By: Claude Opus 5 (1M context) --- .../python/utils/update/test_repository.py | 46 ++++++++++ utils/update/repository.py | 86 ++++++++++++------- 2 files changed, 101 insertions(+), 31 deletions(-) diff --git a/tests/unit/python/utils/update/test_repository.py b/tests/unit/python/utils/update/test_repository.py index 28dfa766c2..b1d6ab0330 100644 --- a/tests/unit/python/utils/update/test_repository.py +++ b/tests/unit/python/utils/update/test_repository.py @@ -9,6 +9,7 @@ from utils.update.repository import ( RepositoryRefEntry, RepositoryRefUpdate, apply_updates, + collect_entries, suppressed_ref_lines, update_config_refs, walk_repo_ref_pairs, @@ -216,5 +217,50 @@ class TestApplyUpdates(unittest.TestCase): self.assertEqual(read_text(str(config_b)), " ref: v2.1.0\n") +class TestCollectEntriesCoversAddons(unittest.TestCase): + def _role(self, root: Path) -> Path: + role = root / "roles" / "web-app-example" + (role / "meta" / "addons").mkdir(parents=True) + return role + + def test_addon_ref_is_collected_beside_the_service_ref(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + role = self._role(root) + (role / "meta" / "services.yml").write_text( + "app:\n" + " repository: https://example.test/app.git\n" + " ref: v1.0.0\n", + encoding="utf-8", + ) + (role / "meta" / "addons" / "plug.yml").write_text( + "config:\n" + " repository: https://example.test/plug.git\n" + " ref: v2.0.0\n", + encoding="utf-8", + ) + + by_ref = {entry.ref: entry for entry in collect_entries(root)} + + self.assertEqual(set(by_ref), {"v1.0.0", "v2.0.0"}) + addon = by_ref["v2.0.0"] + self.assertEqual(addon.entity_path[:2], ("addons", "plug")) + self.assertEqual(addon.config_path.name, "plug.yml") + + def test_addon_ref_honours_its_own_suppression_marker(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + role = self._role(root) + (role / "meta" / "addons" / "plug.yml").write_text( + "config:\n" + " repository: https://example.test/plug.git\n" + " # nocheck: repository-version\n" + " ref: v2.0.0\n", + encoding="utf-8", + ) + + self.assertEqual(collect_entries(root), []) + + if __name__ == "__main__": unittest.main() diff --git a/utils/update/repository.py b/utils/update/repository.py index ac5561fb22..c43f446461 100644 --- a/utils/update/repository.py +++ b/utils/update/repository.py @@ -30,7 +30,7 @@ from typing import TYPE_CHECKING from utils.annotations.suppress import is_suppressed_at from utils.cache.files import read_text from utils.cache.yaml import load_yaml_any -from utils.roles.mapping import ROLE_FILE_META_SERVICES +from utils.roles.mapping import ROLE_DIR_META_ADDONS, ROLE_FILE_META_SERVICES from utils.update.base import ( is_semver, latest_semver, @@ -148,43 +148,67 @@ def suppressed_ref_lines(config_path: Path) -> set[int]: return suppressed +def _entries_of_file( + role_name: str, config_path: Path, entity_prefix: tuple[str, ...] +) -> list[RepositoryRefEntry]: + """Return the unsuppressed semver ``(repository, ref)`` entries of one + declaration file. + + Args: + role_name: role the file belongs to, carried into each entry. + config_path: YAML file to read. + entity_prefix: path segments prepended to the entity path, so an + addon's entries stay distinguishable from a service's. + """ + data = load_yaml_any(str(config_path), default_if_missing=None) + if not isinstance(data, dict): + return [] + + ref_line_index: dict[str, list[int]] = {} + for line_no, ref_value in _ref_lines(read_text(str(config_path)).splitlines()): + ref_line_index.setdefault(ref_value, []).append(line_no) + + suppressed = suppressed_ref_lines(config_path) + entries: list[RepositoryRefEntry] = [] + for entity_path, repo, ref in walk_repo_ref_pairs(data, entity_prefix): + if not is_semver(ref): + continue + candidates = ref_line_index.get(ref) or [] + if not candidates: + continue + line_no = candidates.pop(0) + if line_no in suppressed: + continue + entries.append( + RepositoryRefEntry( + role=role_name, + entity_path=entity_path, + repository=repo, + ref=ref, + config_path=config_path, + line=line_no, + ) + ) + return entries + + def collect_entries(repo_root: Path) -> list[RepositoryRefEntry]: """Return every semver-checkable ``(repository, ref)`` declaration - under ``roles/*/meta/services.yml`` whose ref is not suppressed.""" + under ``roles/*/meta/services.yml`` and ``roles/*/meta/addons/*.yml`` + whose ref is not suppressed.""" roles_root = repo_root / "roles" entries: list[RepositoryRefEntry] = [] for role_dir in sorted(p for p in roles_root.iterdir() if p.is_dir()): services_path = role_dir / ROLE_FILE_META_SERVICES - if not services_path.is_file(): - continue - data = load_yaml_any(str(services_path), default_if_missing=None) - if not isinstance(data, dict): - continue + if services_path.is_file(): + entries += _entries_of_file(role_dir.name, services_path, ()) - raw = read_text(str(services_path)) - ref_line_index: dict[str, list[int]] = {} - for line_no, ref_value in _ref_lines(raw.splitlines()): - ref_line_index.setdefault(ref_value, []).append(line_no) - - suppressed = suppressed_ref_lines(services_path) - for entity_path, repo, ref in walk_repo_ref_pairs(data, ()): - if not is_semver(ref): - continue - candidates = ref_line_index.get(ref) or [] - if not candidates: - continue - line_no = candidates.pop(0) - if line_no in suppressed: - continue - entries.append( - RepositoryRefEntry( - role=role_dir.name, - entity_path=entity_path, - repository=repo, - ref=ref, - config_path=services_path, - line=line_no, - ) + addons_dir = role_dir / ROLE_DIR_META_ADDONS + if not addons_dir.is_dir(): + continue + for addon_path in sorted(addons_dir.glob("*.yml")): + entries += _entries_of_file( + role_dir.name, addon_path, ("addons", addon_path.stem) ) return entries From f2b214c815ab792fef9756ea9f3707523196fc33 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 17:30:57 +0200 Subject: [PATCH 007/197] feat(lint): accept meta/addons as a repository declaration site The guard recognised meta/services.yml as the only place a git URL may be declared. 84a4b04f0 made meta/addons/.yml the one place a Discourse plugin pins itself, so the addon files now carry the clone URL literally, and _iter_role_files picks up every .yml under the role: each declaration reported itself as an undeclared reference. Two halves. The declaration set is now the union over meta/services.yml and every meta/addons/*.yml, so an addon pin legitimises that URL for the whole role, not just for the file holding it. Path.glob on a missing meta/addons yields nothing, so roles without addons need no special case. The self-skip that kept services.yml out of the scan is now a membership test against exactly those declaration paths. Scope stays per role: declared is rebuilt inside the role loop, so a declaration in one role legitimises nothing in another. The failure message and the docstring name both sites, otherwise they send the next reader to the wrong file. This is the test-side counterpart of e4141cb826, which taught collect_entries to see the same addon refs. Co-Authored-By: Claude Opus 5 (1M context) --- .../repositories/test_declared_in_services.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/integration/roles/meta/services/repositories/test_declared_in_services.py b/tests/integration/roles/meta/services/repositories/test_declared_in_services.py index f79749ee30..d3336c8d5d 100644 --- a/tests/integration/roles/meta/services/repositories/test_declared_in_services.py +++ b/tests/integration/roles/meta/services/repositories/test_declared_in_services.py @@ -1,6 +1,7 @@ """Integration guard: every git-clone URL referenced inside a role (Dockerfile, tasks, templates, files) MUST be declared as a -`repository:` value in that role's ``meta/services.yml``. +`repository:` value in that role's ``meta/services.yml`` or in one of +its ``meta/addons/.yml`` files, which is where an addon pins itself. The same applies symmetrically to git refs: any `compose_repository_ref` or `--branch ` (clone) value that ties to a declared @@ -39,7 +40,7 @@ import unittest from utils.cache.files import iter_project_files, read_text from utils.cache.yaml import load_yaml_any -from utils.roles.mapping import ROLE_FILE_META_SERVICES +from utils.roles.mapping import ROLE_DIR_META_ADDONS, ROLE_FILE_META_SERVICES from . import PROJECT_ROOT @@ -106,18 +107,20 @@ class TestDeclaredInServices(unittest.TestCase): for role_dir in sorted(p for p in ROLES_DIR.iterdir() if p.is_dir()): role_name = role_dir.name services_path = role_dir / ROLE_FILE_META_SERVICES - declared = ( - _collect_declared_repositories(services_path) - if services_path.is_file() - else set() - ) + declaration_paths = [services_path] + declaration_paths += sorted((role_dir / ROLE_DIR_META_ADDONS).glob("*.yml")) + declared: set[str] = set() + for declaration_path in declaration_paths: + if declaration_path.is_file(): + declared |= _collect_declared_repositories(declaration_path) + declaration_names = {str(p) for p in declaration_paths} seen_paths: set[str] = set() for fpath in _iter_role_files(role_dir): if fpath in seen_paths: continue seen_paths.add(fpath) - if fpath.endswith("/" + ROLE_FILE_META_SERVICES): + if fpath in declaration_names: continue try: text = read_text(fpath) @@ -140,7 +143,8 @@ class TestDeclaredInServices(unittest.TestCase): f"{role_name}: {rel}:{line_no} " f"references `{url}` but it is not declared " f"as `repository:` in " - f"{role_name}/{ROLE_FILE_META_SERVICES}" + f"{role_name}/{ROLE_FILE_META_SERVICES} " + f"or {role_name}/{ROLE_DIR_META_ADDONS}/" ) if offenders: From ea46bf5fde07772e2c8e5e565952057c9839850d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 17:34:23 +0200 Subject: [PATCH 008/197] refactor(discourse): drop the plugin repository vars nothing reads DISCOURSE_LDAP_AUTH_PLUGIN_REPOSITORY and DISCOURSE_LDAP_AUTH_PLUGIN_REF date from fd22d34559, when every role mirrored its git URL into a per-role REPOSITORY/REF pair. 84a4b04f0 pointed the clone loop in config.yml.j2 at plugin_config.config.repository and plugin_config.config.ref instead, reading the addon declaration directly, and left the mirror without a reader. They are not merely noise. Both are unconditional top-level vars whose lookup('config', ...) carries no fourth argument and is therefore strict, and both address a path that exists only while this one addon keeps this one id. Renaming or removing discourse-ldap-auth would have failed every Discourse task at resolution time over a value no caller wanted. The surviving LDAP path is untouched: DISCOURSE_LDAP_ENABLED reads services.ldap.enabled, and stack_host/02_credentials.yml takes the site settings through DISCOURSE_PLUGINS['discourse-ldap-auth'].config.site_settings. A tree-wide grep for DISCOURSE_LDAP_AUTH_PLUGIN now returns nothing. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-discourse/vars/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/web-app-discourse/vars/main.yml b/roles/web-app-discourse/vars/main.yml index e56af0c86f..8e423d3f92 100644 --- a/roles/web-app-discourse/vars/main.yml +++ b/roles/web-app-discourse/vars/main.yml @@ -18,8 +18,6 @@ DISCOURSE_REDIS_PASSWORD: "{{ lookup('engine', 'redis', application_id, 'passwor DISCOURSE_REPOSITORY_URL: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.repository') }}" DISCOURSE_REF: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.ref') }}" DISCOURSE_VERSION: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.application.ref') }}" -DISCOURSE_LDAP_AUTH_PLUGIN_REPOSITORY: "{{ lookup('config', application_id, 'addons.discourse-ldap-auth.config.repository') }}" -DISCOURSE_LDAP_AUTH_PLUGIN_REF: "{{ lookup('config', application_id, 'addons.discourse-ldap-auth.config.ref') }}" DISCOURSE_DATABASE: "{{ lookup('database', application_id ) }}" DISCOURSE_DATABASE_TYPE: "{{ DISCOURSE_DATABASE.type }}" From 4a0fca2bcb2193455cdbef70006370d9116d05a1 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 17:35:49 +0200 Subject: [PATCH 009/197] test(update): build the addon fixture from the role-file constants test_file_spot failed the gate on the pathlib chain that spelled meta/services.yml out by hand. Both paths now come from utils.roles.mapping as ROLE_FILE_META_SERVICES and ROLE_DIR_META_ADDONS, following ea37027f49 and 8243aa571b. All four chains move, not only the reported one. The lint reads the role-file shape and lets a "meta" / "addons" directory chain through, so leaving those behind would have split one fixture across two sources of truth, which is the split the lint exists to prevent. The constants are slash-bearing strings that Path accepts as a single segment, so the paths are unchanged. Only the throwaway role built under tempfile in TestCollectEntriesCoversAddons is affected; its assertions on entity_path and config_path.name are untouched. autoformat then reflowed the two write_text concatenations, which fit on one line once the names shortened, and the generator in test_oidc_setting_order.py. Co-Authored-By: Claude Opus 5 (1M context) --- .../test_oidc_setting_order.py | 4 +--- .../unit/python/utils/update/test_repository.py | 17 +++++++---------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/tests/unit/python/roles/web-app-discourse/test_oidc_setting_order.py b/tests/unit/python/roles/web-app-discourse/test_oidc_setting_order.py index e619fb2bb4..87ef01cba5 100644 --- a/tests/unit/python/roles/web-app-discourse/test_oidc_setting_order.py +++ b/tests/unit/python/roles/web-app-discourse/test_oidc_setting_order.py @@ -14,9 +14,7 @@ PREREQUISITES = ( def _line_of(lines: list[str], setting: str) -> int: return next( - index - for index, line in enumerate(lines) - if f"SiteSetting.{setting} = " in line + index for index, line in enumerate(lines) if f"SiteSetting.{setting} = " in line ) diff --git a/tests/unit/python/utils/update/test_repository.py b/tests/unit/python/utils/update/test_repository.py index b1d6ab0330..a956b4354e 100644 --- a/tests/unit/python/utils/update/test_repository.py +++ b/tests/unit/python/utils/update/test_repository.py @@ -5,6 +5,7 @@ import unittest from pathlib import Path from utils.cache.files import read_text +from utils.roles.mapping import ROLE_DIR_META_ADDONS, ROLE_FILE_META_SERVICES from utils.update.repository import ( RepositoryRefEntry, RepositoryRefUpdate, @@ -220,23 +221,19 @@ class TestApplyUpdates(unittest.TestCase): class TestCollectEntriesCoversAddons(unittest.TestCase): def _role(self, root: Path) -> Path: role = root / "roles" / "web-app-example" - (role / "meta" / "addons").mkdir(parents=True) + (role / ROLE_DIR_META_ADDONS).mkdir(parents=True) return role def test_addon_ref_is_collected_beside_the_service_ref(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) role = self._role(root) - (role / "meta" / "services.yml").write_text( - "app:\n" - " repository: https://example.test/app.git\n" - " ref: v1.0.0\n", + (role / ROLE_FILE_META_SERVICES).write_text( + "app:\n repository: https://example.test/app.git\n ref: v1.0.0\n", encoding="utf-8", ) - (role / "meta" / "addons" / "plug.yml").write_text( - "config:\n" - " repository: https://example.test/plug.git\n" - " ref: v2.0.0\n", + (role / ROLE_DIR_META_ADDONS / "plug.yml").write_text( + "config:\n repository: https://example.test/plug.git\n ref: v2.0.0\n", encoding="utf-8", ) @@ -251,7 +248,7 @@ class TestCollectEntriesCoversAddons(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) role = self._role(root) - (role / "meta" / "addons" / "plug.yml").write_text( + (role / ROLE_DIR_META_ADDONS / "plug.yml").write_text( "config:\n" " repository: https://example.test/plug.git\n" " # nocheck: repository-version\n" From ce8368019dbfd8c328edcf14df2a383f0710bcd3 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 18:10:40 +0200 Subject: [PATCH 010/197] perf(constructor): carry the applications SPOT through the merge block 113113e553 parks the carrier after the token store, 165 lines below the merge block, so each of the block's six tasks still renders the applications SPOT cold in its own forked worker. Measured in CI run wp-swarm-tor2 they cost 323.51s in the PASS 1 main play, 180.39s in the backup play and 479.79s in PASS 2. The cost is the render, not the work. Two tasks in that block share the exact construct `defaults_X | combine(X | default({}, true), recursive=True)`: design takes 0.247s and service_provider 75.94s. The difference is that defaults_service_provider is dense with lookup('config'/'users'/'domains'/'tls') and every one of them lands in get_merged_applications. The carrier cannot simply move up. get_merged_applications serves the carrier fact whenever the key matches, and the carrier action goes through that same path, so a second carrier task would be handed the first one's payload instead of rendering. merged_applications_cache_key reads neither group_names nor the token file, and with a real templar 71 of 265 applications differ by group_names alone, so a carrier left standing across add_host or the token store would be served where a fresh render belongs. So the block opens its own carrier and drops it again at the end. Nothing outside the block changes: every task from sys-version to the stage-wide carrier renders exactly as before. Placement follows measurement. A per-key perturbation test over the whole 265-application payload, with the credential regeneration noise floor subtracted and every role in group_names, finds exactly two reads of the seven facts the block sets: .web-app-matomo.services.matomo.excluded_ips[0] <- networks .web-app-wordpress.services.wordpress.title <- service_provider networks is the block's first task and service_provider its last, which is why the carrier opens after the one and is dropped before the other takes effect. The block's tasks already render before service_provider is merged today, so the frozen payload is what they would have rendered anyway. The action gains a `clear` argument so the playbook never spells the private fact name. Parking the carrier serves a payload in 0.000s against a 3.417s cold render; a wrong key and a cleared fact both force the render again, which is what makes the drop task work. Worth roughly 806s of the 20199s job, against a floor of 673.6s. Co-Authored-By: Claude Opus 5 (1M context) --- plugins/action/applications_carrier.py | 36 ++++++++++++++--- tasks/stages/01_constructor.yml | 7 ++++ .../action/test_applications_carrier.py | 40 +++++++++++++++++++ 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/plugins/action/applications_carrier.py b/plugins/action/applications_carrier.py index 5991ff5cd8..f7711ed295 100644 --- a/plugins/action/applications_carrier.py +++ b/plugins/action/applications_carrier.py @@ -5,19 +5,31 @@ Ansible forks a fresh worker per task, so the in-process render cache of a place all later workers inherit. This action renders in the worker of one constructor task and returns the payload as a host fact; every later worker inherits that fact at fork time and ``get_merged_applications`` serves it -instead of rendering. Sits after the token store in -``tasks/stages/01_constructor.yml`` so the embedded user records already -carry the loaded tokens:: +instead of rendering:: - name: ๐ŸงŠ Render the applications SPOT once and carry it into every task's worker applications_carrier: {} +``tasks/stages/01_constructor.yml`` parks two of them. The stage-wide one +sits after the token store so the embedded user records already carry the +loaded tokens, and after the ``add_host`` tasks because 114 roles gate +services on ``group_names`` and the payload bakes those in. The +merge block opens an earlier, block-scoped one and drops it again at the +block's end, because ``merged_applications_cache_key`` reads neither +``group_names`` nor the token file: a carrier left standing across either +would be served unchanged where a fresh render belongs:: + + - name: ๐ŸงŠ Drop the merge-block carrier so later stages re-render + applications_carrier: + clear: true + The result is masked by the action itself: the fact is the whole rendered tree, about 0.4 MB with every application's credentials, and printing it at ``-v`` is neither credential debugging nor readable. Args: - none. Any argument is an error. + clear: drop the carrier fact instead of rendering. Defaults to false. + Any other argument is an error. """ from __future__ import annotations @@ -25,6 +37,7 @@ from __future__ import annotations from typing import Any from ansible.errors import AnsibleActionFail +from ansible.module_utils.parsing.convert_bool import boolean from ansible.plugins.action import ActionBase from ansible.plugins.loader import lookup_loader @@ -39,8 +52,19 @@ class ActionModule(ActionBase): ) -> dict[str, Any]: result = super().run(tmp, task_vars) task_vars = task_vars or {} - if self._task.args: - raise AnsibleActionFail("applications_carrier takes no arguments") + args = self._task.args or {} + unexpected = sorted(set(args) - {"clear"}) + if unexpected: + raise AnsibleActionFail( + f"applications_carrier: unknown argument(s) {', '.join(unexpected)}; " + "the only argument is 'clear'" + ) + + if boolean(args.get("clear", False)): + result.update( + changed=False, ansible_facts={APPLICATIONS_RENDERED_FACT: None} + ) + return result lookup = lookup_loader.get( "applications", loader=self._loader, templar=self._templar diff --git a/tasks/stages/01_constructor.yml b/tasks/stages/01_constructor.yml index 70a1a2aab4..5f6b5c33e5 100644 --- a/tasks/stages/01_constructor.yml +++ b/tasks/stages/01_constructor.yml @@ -28,6 +28,9 @@ ansible.builtin.set_fact: networks: "{{ defaults_networks | combine(networks | default({}, true), recursive=True) }}" + - name: ๐ŸงŠ Render the applications SPOT once for the merge block + applications_carrier: {} + - name: "โœ… Sanity check: all group_names must exist in applications" ansible.builtin.assert: that: @@ -96,6 +99,10 @@ ansible.builtin.set_fact: service_provider: "{{ defaults_service_provider | combine(service_provider | default({}, true), recursive=True) }}" + - name: ๐ŸงŠ Drop the merge-block carrier so later stages re-render + applications_carrier: + clear: true + - name: "๐Ÿ’พ Persist {{ SOFTWARE_NAME }} version" ansible.builtin.include_role: name: sys-version diff --git a/tests/unit/python/plugins/action/test_applications_carrier.py b/tests/unit/python/plugins/action/test_applications_carrier.py index 5e80413bbb..841b2eea44 100644 --- a/tests/unit/python/plugins/action/test_applications_carrier.py +++ b/tests/unit/python/plugins/action/test_applications_carrier.py @@ -68,6 +68,46 @@ class TestApplicationsCarrierActionPlugin(unittest.TestCase): ): action.run(task_vars={}) + def test_clear_drops_the_fact_without_rendering(self): + lookup = _FakeLookup({"key": ["roles", []], "applications": {}}) + action = _make_action(_FakeTask(args={"clear": True})) + with ( + patch( + "plugins.action.applications_carrier.ActionBase.run", + autospec=True, + return_value={}, + ), + patch( + "plugins.action.applications_carrier.lookup_loader.get", + return_value=lookup, + ), + ): + result = action.run(task_vars={"applications": {}}) + + self.assertIsNone(result["ansible_facts"][APPLICATIONS_RENDERED_FACT]) + self.assertFalse(result["changed"]) + self.assertEqual(lookup.calls, []) + + def test_clear_false_still_renders(self): + carrier = {"key": ["roles", ["a", "b", "c", "d"]], "applications": {"x": {}}} + lookup = _FakeLookup(carrier) + action = _make_action(_FakeTask(args={"clear": "no"})) + with ( + patch( + "plugins.action.applications_carrier.ActionBase.run", + autospec=True, + return_value={}, + ), + patch( + "plugins.action.applications_carrier.lookup_loader.get", + return_value=lookup, + ), + ): + result = action.run(task_vars={"applications": {}}) + + self.assertIs(result["ansible_facts"][APPLICATIONS_RENDERED_FACT], carrier) + self.assertEqual(len(lookup.calls), 1) + if __name__ == "__main__": unittest.main() From dac61edbc6411c66a604d17931a0cd5067ac70d2 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 19:21:47 +0200 Subject: [PATCH 011/197] feat(ai): let LM Studio serve the models Ollama serves, under one catalogue svc-ai-lmstudio had no model-load step at all: tasks/00_core.yml set the once-flag and the container network, nothing else. The gateway published lmstudio/default anyway, so the route existed and nothing could answer it. wp-compose-tor.log:48994 shows the route written into the config, :51770 shows the warm step reaching qwen2.5:0.5b on Ollama instead, and the green ai-engine spec ran against that. LM Studio was deployed, published and never prompted. The pre-pull mirrors the one that works. tasks/01_manager_ops.yml resolves the hosting node and container id, loops the declared models through `lms get --yes --gguf`, overlaps them with async/poll and reaps before playwright, exactly like svc-ai-ollama. That `lms` exists is measured, not assumed: /app/daemon-run.sh:67 in the image starts ./.bundle/lms server start, and `lms get --help` documents -y for scripting and --gguf for the format. Synonymity needs both namespaces in one place. Ollama knows qwen2.5:0.5b, LM Studio knows qwen2.5-0.5b-instruct, so AI_LOCAL_MODELS carries both per entry: `alias` is what the gateway publishes and what Ollama pulls, `name` is what `lms get` resolves. Both roles now reference that one list, and config.yaml.j2 publishes each backend under the alias, routing Ollama to ollama/ and LM Studio to openai/. The lmstudio loop skips an alias Ollama already publishes, so two backends never produce a duplicate model_name and Ollama keeps precedence. lmstudio/default is gone; the model name is now the same whichever backend answers it. Ollama's preload_models changes shape from a list of strings to the {alias, name} entries. Every consumer moves with it: the pull command, the template loop, and both expressions in 16_ai.yml. An inventory that overrides `applications.svc-ai-ollama.services.ollama.preload_models` with the old string list will break at map(attribute='alias'); this is a deliberate shape change, in the spirit of 7b15c61dfc. The CI model list is not a property of an installation, so AI_LOCAL_MODELS_CI lives in inventories/development/default.yml, whose own header scopes that file to the cross-cutting environment knobs. test_variable_definitions only knew group_vars, so it reported the constant as undefined; the shipped inventories are a definition source too, because the deploy CLI bakes them into host_vars, and the guard's own docstring already claimed "defined somewhere in the repository". A latent crash came out of this: `| list | first` raises on an empty sequence, so a backend deployed without preload models blew up LITELLM_CHAT_MODEL. Ollama with only embedding models could already hit it; the existing test dodged it by rendering only _SERVED. `| default('')` now yields the empty string that _SERVED already reports as false. Tests: config.yaml.j2 is rendered rather than read, over ollama-only, lmstudio-only, both and neither, asserting the published model_name set, that both backends name the same alias, that a shared alias is published once and routed to Ollama, and that the render is valid YAML. test_litellm_chat_model gains the lmstudio branch, the no-preload case and the same-model assertion. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/16_ai.yml | 20 ++- inventories/development/default.yml | 4 + roles/svc-ai-litellm/templates/config.yaml.j2 | 15 ++- roles/svc-ai-lmstudio/README.md | 5 +- roles/svc-ai-lmstudio/meta/services.yml | 2 + roles/svc-ai-lmstudio/meta/variants.yml | 1 + roles/svc-ai-lmstudio/tasks/00_core.yml | 4 + .../svc-ai-lmstudio/tasks/01_manager_ops.yml | 41 ++++++ roles/svc-ai-lmstudio/vars/main.yml | 2 + roles/svc-ai-ollama/meta/services.yml | 5 +- roles/svc-ai-ollama/meta/variants.yml | 3 +- roles/svc-ai-ollama/tasks/01_manager_ops.yml | 3 +- .../jinja/test_variable_definitions.py | 19 ++- .../group_vars/test_litellm_chat_model.py | 61 +++++++-- .../svc-ai-litellm/templates/__init__.py | 3 + .../templates/test_config_model_list.py | 127 ++++++++++++++++++ 16 files changed, 278 insertions(+), 37 deletions(-) create mode 100644 roles/svc-ai-lmstudio/tasks/01_manager_ops.yml create mode 100644 tests/unit/python/roles/svc-ai-litellm/templates/__init__.py create mode 100644 tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index 6f08d29595..63ce93dc2c 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -13,14 +13,26 @@ LITELLM_OPENAI_BASE_LOCAL_URL: "{{ LITELLM_BASE_LOCAL_URL }}/v1" LITELLM_BACKEND_ROLES: "{{ lookup('deployment').inventory }}" -LITELLM_CHAT_MODEL: "{{ (lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') | reject('search', 'embed') | list | first) if 'svc-ai-ollama' in LITELLM_BACKEND_ROLES - else ('lmstudio/default' if 'svc-ai-lmstudio' in LITELLM_BACKEND_ROLES else ('openrouter/auto' if ((OPENROUTER_API_KEY | string) | length > 0) else '')) }}" +AI_LOCAL_MODELS: + - alias: llama3:latest + name: llama-3.2-3b-instruct + - alias: mistral:latest + name: mistral-7b-instruct-v0.3 + - alias: nomic-embed-text:latest + name: nomic-embed-text-v1.5 + +LITELLM_CHAT_MODEL: "{{ (lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') | map(attribute='alias') | reject('search', 'embed') | list | first | + default('')) if 'svc-ai-ollama' in LITELLM_BACKEND_ROLES else ((lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') | map(attribute='alias') + | reject('search', 'embed') | list | first | default('')) if 'svc-ai-lmstudio' in LITELLM_BACKEND_ROLES else ('openrouter/auto' if ((OPENROUTER_API_KEY | string) + | length > 0) else '')) }}" LITELLM_CHAT_MODEL_SERVED: >- {{ (('svc-ai-ollama' in LITELLM_BACKEND_ROLES) and ((lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') - | reject('search', 'embed') | list) | length > 0)) - or ('svc-ai-lmstudio' in LITELLM_BACKEND_ROLES) + | map(attribute='alias') | reject('search', 'embed') | list) | length > 0)) + or (('svc-ai-lmstudio' in LITELLM_BACKEND_ROLES) + and ((lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') + | map(attribute='alias') | reject('search', 'embed') | list) | length > 0)) or ((OPENROUTER_API_KEY | string) | length > 0) }} OPENROUTER_API_KEY: "" diff --git a/inventories/development/default.yml b/inventories/development/default.yml index 61421409f4..b7aa882ca4 100644 --- a/inventories/development/default.yml +++ b/inventories/development/default.yml @@ -35,6 +35,10 @@ design: colors: base: "#FFA500" +AI_LOCAL_MODELS_CI: + - alias: qwen2.5:0.5b + name: qwen2.5-0.5b-instruct + OIDC_OVERRIDE: CLIENT: SECRET: VERYUNSAFESAMPLEPASSWORD1234 diff --git a/roles/svc-ai-litellm/templates/config.yaml.j2 b/roles/svc-ai-litellm/templates/config.yaml.j2 index 1b55fbc39e..07faa6deec 100644 --- a/roles/svc-ai-litellm/templates/config.yaml.j2 +++ b/roles/svc-ai-litellm/templates/config.yaml.j2 @@ -1,25 +1,26 @@ {% set _ollama_models = lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') if 'svc-ai-ollama' in group_names else [] %} -{% set has_backend = _ollama_models or ('svc-ai-lmstudio' in group_names) or LITELLM_OPENROUTER_API_KEY %} +{% set _lmstudio_models = lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') if 'svc-ai-lmstudio' in group_names else [] %} +{% set has_backend = _ollama_models or _lmstudio_models or LITELLM_OPENROUTER_API_KEY %} model_list:{{ ' []' if not has_backend else '' }} {% for _model in _ollama_models %} - - model_name: "{{ _model }}" + - model_name: "{{ _model.alias }}" litellm_params: - model: "ollama/{{ _model }}" + model: "ollama/{{ _model.alias }}" api_base: {{ OLLAMA_BASE_LOCAL_URL }} rpm: 1000 max_tokens: {{ LITELLM_MAX_OUTPUT_TOKENS }} timeout: {{ LITELLM_UPSTREAM_TIMEOUT }} {% endfor %} -{% if 'svc-ai-lmstudio' in group_names %} - - model_name: lmstudio/default +{% for _model in _lmstudio_models if _model.alias not in (_ollama_models | map(attribute='alias') | list) %} + - model_name: "{{ _model.alias }}" litellm_params: - model: openai/local-model + model: "openai/{{ _model.name }}" api_base: {{ LMSTUDIO_BASE_LOCAL_URL }}/v1 api_key: lm-studio rpm: 1000 max_tokens: {{ LITELLM_MAX_OUTPUT_TOKENS }} timeout: {{ LITELLM_UPSTREAM_TIMEOUT }} -{% endif %} +{% endfor %} {% if LITELLM_OPENROUTER_API_KEY %} - model_name: openrouter/auto litellm_params: diff --git a/roles/svc-ai-lmstudio/README.md b/roles/svc-ai-lmstudio/README.md index 4e311ba7c7..d313883f0a 100644 --- a/roles/svc-ai-lmstudio/README.md +++ b/roles/svc-ai-lmstudio/README.md @@ -6,7 +6,7 @@ ## Overview -This role deploys LM Studio as a headless model server in a single container, in both Docker Compose and Docker Swarm deployments. The server listens on port 1234 on the internal container network and is not published through the reverse proxy, while downloaded models and server settings persist in a dedicated volume. Deployed together with the LiteLLM Gateway, the instance is reachable through the gateway under the model alias `lmstudio/default`, next to or instead of the Ollama backend. +This role deploys LM Studio as a headless model server in a single container, in both Docker Compose and Docker Swarm deployments. The server listens on port 1234 on the internal container network and is not published through the reverse proxy, while downloaded models and server settings persist in a dedicated volume. The role downloads its declared models at deploy time and the LiteLLM Gateway publishes them under the same aliases Ollama uses, so a consumer asks for one model name and reaches whichever backend the deployment provides. ## Cosmos @@ -35,7 +35,8 @@ Solid `1:1` edges are fixed relationships; dashed `0..1` edges are conditional ( - **OpenAI-compatible endpoint:** The headless server answers `/v1` requests on port 1234 inside the container network. - **CPU inference:** The role pins the CPU build of the upstream image and passes no GPU device into the container. - **Persistent model store:** A named volume mounted at `/root/.lmstudio` keeps downloaded models and server settings across redeploys. -- **Gateway backend:** The LiteLLM Gateway routes the model alias `lmstudio/default` to this instance when both roles are deployed together, as one of the interchangeable local backends alongside Ollama. +- **Preloaded models:** Every entry of `services.lmstudio.preload_models` is downloaded on the hosting node with `lms get`, overlapped and reaped like the Ollama pre-pull. Each entry carries the LM Studio `name` to download and the `alias` the gateway publishes it under. +- **Gateway backend:** The LiteLLM Gateway publishes those aliases, which are the names Ollama serves too, so the same model name routes to whichever local backend a deployment runs. - **Bounded resources:** The container is capped at 4 CPUs, 8 GB of memory and 2048 processes, and the role declares a minimum of 588 MB free storage for the image; downloaded models grow the volume beyond that. - **Backup integration:** Backup Docker Volumes snapshots the model volume when that role is present, without stopping the container. diff --git a/roles/svc-ai-lmstudio/meta/services.yml b/roles/svc-ai-lmstudio/meta/services.yml index d3471ca63e..b9a3f271ed 100644 --- a/roles/svc-ai-lmstudio/meta/services.yml +++ b/roles/svc-ai-lmstudio/meta/services.yml @@ -22,6 +22,8 @@ lmstudio: ports: internal: http: 1234 + preload_models: "{{ AI_LOCAL_MODELS }}" + preload_async_time: 3600 lifecycle: beta # nocheck: playwright-service-flag, dynamic-flag diff --git a/roles/svc-ai-lmstudio/meta/variants.yml b/roles/svc-ai-lmstudio/meta/variants.yml index 5a12caf234..5ffa7b228c 100644 --- a/roles/svc-ai-lmstudio/meta/variants.yml +++ b/roles/svc-ai-lmstudio/meta/variants.yml @@ -6,5 +6,6 @@ container_backup: enabled: true lmstudio: + preload_models: "{{ AI_LOCAL_MODELS_CI }}" mem_reservation: 1g mem_limit: 2g diff --git a/roles/svc-ai-lmstudio/tasks/00_core.yml b/roles/svc-ai-lmstudio/tasks/00_core.yml index bd183d0e32..16942c364d 100644 --- a/roles/svc-ai-lmstudio/tasks/00_core.yml +++ b/roles/svc-ai-lmstudio/tasks/00_core.yml @@ -5,3 +5,7 @@ include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/network/routine.yml') }}" vars: compose_handlers_flush: true + +- name: "โฌ‡๏ธ Manager-side model pre-pull on the hosting node" + ansible.builtin.include_tasks: 01_manager_ops.yml + when: IS_STACK_HOST | bool diff --git a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml new file mode 100644 index 0000000000..d667dca141 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml @@ -0,0 +1,41 @@ +--- +- name: "๐Ÿ” Resolve {{ application_id }} hosting node + container id" + ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml') }}" + vars: + resolve_service_key: "{{ entity_name }}" + resolve_output_prefix: LMSTUDIO + resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" + +- name: โฌ‡๏ธ Pre-pull LM Studio models + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} lms get --yes --gguf {{ model.name | quote }} + register: lmstudio_pull_result + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.alias }}" + # rationale: async; model downloads run overlapped, results reaped below before playwright + async: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" + poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" + retries: "{{ 0 if ASYNC_ENABLED | bool else 5 }}" + delay: 15 + until: > + (lmstudio_pull_result.rc | default(0)) == 0 or + ('already downloaded' in (lmstudio_pull_result.stdout | default(''))) + changed_when: false + failed_when: >- + (lmstudio_pull_result.failed | default(false)) + and 'already downloaded' not in ((lmstudio_pull_result.stdout | default('')) ~ (lmstudio_pull_result.stderr | default(''))) + delegate_to: "{{ LMSTUDIO_HOST_NODE }}" + +- name: "โณ Reap the LM Studio model pre-pulls" + ansible.builtin.include_tasks: + file: utils/async/reap.yml + apply: + delegate_to: "{{ LMSTUDIO_HOST_NODE }}" + vars: + async_reap_jobs: "{{ lmstudio_pull_result.results | default([]) }}" + async_reap_label: "LM Studio model pre-pull" + async_reap_tolerate: + - "already downloaded" + ASYNC_TIME: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME }}" diff --git a/roles/svc-ai-lmstudio/vars/main.yml b/roles/svc-ai-lmstudio/vars/main.yml index cea5302881..ce7a2585e4 100644 --- a/roles/svc-ai-lmstudio/vars/main.yml +++ b/roles/svc-ai-lmstudio/vars/main.yml @@ -4,3 +4,5 @@ entity_name: "{{ application_id | get_entity_name }}" LMSTUDIO_CONTAINER: "{{ lookup('config', application_id, 'services.lmstudio.name') }}" LMSTUDIO_PORT: "{{ lookup('config', application_id, 'services.lmstudio.ports.internal.http') }}" +LMSTUDIO_PRELOAD_MODELS: "{{ lookup('config', application_id, 'services.lmstudio.preload_models') }}" +LMSTUDIO_PRELOAD_ASYNC_TIME: "{{ lookup('config', application_id, 'services.lmstudio.preload_async_time') }}" diff --git a/roles/svc-ai-ollama/meta/services.yml b/roles/svc-ai-ollama/meta/services.yml index 6b3fda75b0..49d885028a 100644 --- a/roles/svc-ai-ollama/meta/services.yml +++ b/roles/svc-ai-ollama/meta/services.yml @@ -23,10 +23,7 @@ ollama: ports: internal: http: 11434 - preload_models: - - llama3:latest - - mistral:latest - - nomic-embed-text:latest + preload_models: "{{ AI_LOCAL_MODELS }}" preload_async_time: 3600 keep_alive: "-1" lifecycle: beta diff --git a/roles/svc-ai-ollama/meta/variants.yml b/roles/svc-ai-ollama/meta/variants.yml index 43b327aa21..4e89b5aa75 100644 --- a/roles/svc-ai-ollama/meta/variants.yml +++ b/roles/svc-ai-ollama/meta/variants.yml @@ -11,7 +11,6 @@ container_backup: enabled: true ollama: - preload_models: - - qwen2.5:0.5b + preload_models: "{{ AI_LOCAL_MODELS_CI }}" mem_reservation: 1g mem_limit: 2g diff --git a/roles/svc-ai-ollama/tasks/01_manager_ops.yml b/roles/svc-ai-ollama/tasks/01_manager_ops.yml index c192f0272a..11fc3a70fb 100644 --- a/roles/svc-ai-ollama/tasks/01_manager_ops.yml +++ b/roles/svc-ai-ollama/tasks/01_manager_ops.yml @@ -8,11 +8,12 @@ - name: โฌ‡๏ธ Pre-pull Ollama models ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node - container exec -i {{ OLLAMA_LOCAL_CID | quote }} ollama pull {{ model }} + container exec -i {{ OLLAMA_LOCAL_CID | quote }} ollama pull {{ model.alias }} register: pull_result loop: "{{ OLLAMA_PRELOAD_MODELS }}" loop_control: loop_var: model + label: "{{ model.alias }}" # rationale: async; model pulls run overlapped, results reaped below before playwright async: "{{ OLLAMA_PRELOAD_ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" diff --git a/tests/integration/ansible/jinja/test_variable_definitions.py b/tests/integration/ansible/jinja/test_variable_definitions.py index ee269ed5b1..605fd2ebd5 100644 --- a/tests/integration/ansible/jinja/test_variable_definitions.py +++ b/tests/integration/ansible/jinja/test_variable_definitions.py @@ -17,8 +17,12 @@ logger = logging.getLogger(__name__) class TestVariableDefinitions(unittest.TestCase): """ Ensures that every Jinja2 variable used in templates/playbooks is defined - somewhere in the repository (direct var files, set_fact/vars blocks, - loop_var/register names, Jinja set/for definitions, and Jinja macro parameters). + somewhere in the repository (direct var files, the shipped inventories, + set_fact/vars blocks, loop_var/register names, Jinja set/for definitions, + and Jinja macro parameters). + + The inventories count because the deploy CLI bakes them into host_vars, so + a `meta/variants.yml` entry resolves against them exactly like a group_var. If a variable is not defined, the test passes only if a corresponding fallback key exists (either "default_" or "defaults_"). @@ -32,6 +36,7 @@ class TestVariableDefinitions(unittest.TestCase): self.var_files: list[str] = [] roles_prefix = str(Path(self.project_root) / "roles") + "/" group_vars_prefix = str(Path(self.project_root) / "group_vars" / "all") + "/" + inventory_prefix = str(Path(self.project_root) / "inventories") + "/" def _is_role_var_file(p: str) -> bool: if not p.startswith(roles_prefix): @@ -40,9 +45,13 @@ class TestVariableDefinitions(unittest.TestCase): return len(parts) >= 3 and parts[1] in ("vars", "defaults") for p in iter_project_files(extensions=(".yml",)): - if _is_role_var_file(p) or ( - p.startswith(group_vars_prefix) - and "/" not in p[len(group_vars_prefix) :] + if ( + _is_role_var_file(p) + or ( + p.startswith(group_vars_prefix) + and "/" not in p[len(group_vars_prefix) :] + ) + or p.startswith(inventory_prefix) ): self.var_files.append(p) diff --git a/tests/unit/python/group_vars/test_litellm_chat_model.py b/tests/unit/python/group_vars/test_litellm_chat_model.py index 4705bbd14b..4b4022ceb1 100644 --- a/tests/unit/python/group_vars/test_litellm_chat_model.py +++ b/tests/unit/python/group_vars/test_litellm_chat_model.py @@ -11,10 +11,12 @@ from utils.cache.yaml import load_yaml _AI_VARS = Path(PROJECT_ROOT) / "group_vars" / "all" / "16_ai.yml" -def _stub_lookup(preload_models): +def _stub_lookup(preload_models, lmstudio_models): def lookup(kind, role, path, *args): if (kind, path) == ("config", "services.ollama.preload_models"): return preload_models + if (kind, path) == ("config", "services.lmstudio.preload_models"): + return lmstudio_models raise AssertionError(f"unexpected lookup({kind!r}, {role!r}, {path!r})") return lookup @@ -38,13 +40,19 @@ class TestLitellmChatModel(unittest.TestCase): ) cls.source = load_yaml(_AI_VARS) - def _render(self, name, *, roles, api_key, preload_models=()): + def _render(self, name, *, roles, api_key, preload_models=(), lmstudio_models=()): return ( self.env.from_string(self.source[name]) .render( LITELLM_BACKEND_ROLES=list(roles), OPENROUTER_API_KEY=api_key, - lookup=_stub_lookup(list(preload_models)), + lookup=_stub_lookup( + [{"alias": alias, "name": alias} for alias in preload_models], + [ + {"alias": alias, "name": f"{alias}-gguf"} + for alias in lmstudio_models + ], + ), ) .strip() ) @@ -66,10 +74,35 @@ class TestLitellmChatModel(unittest.TestCase): self.assertEqual(served, "True") def test_lmstudio_wins_over_the_openrouter_fallback(self): - model, served = self._both(roles=["svc-ai-lmstudio"], api_key="") - self.assertEqual(model, "lmstudio/default") + model, served = self._both( + roles=["svc-ai-lmstudio"], api_key="", lmstudio_models=["qwen2.5:0.5b"] + ) + self.assertEqual(model, "qwen2.5:0.5b") self.assertEqual(served, "True") + def test_lmstudio_without_a_preloaded_model_serves_nothing(self): + served = self._render( + "LITELLM_CHAT_MODEL_SERVED", roles=["svc-ai-lmstudio"], api_key="" + ) + self.assertEqual(served, "False") + + def test_both_backends_name_the_same_model(self): + alias = "qwen2.5:0.5b" + with_ollama = self._render( + "LITELLM_CHAT_MODEL", + roles=["svc-ai-ollama"], + api_key="", + preload_models=[alias], + ) + with_lmstudio = self._render( + "LITELLM_CHAT_MODEL", + roles=["svc-ai-lmstudio"], + api_key="", + lmstudio_models=[alias], + ) + self.assertEqual(with_ollama, with_lmstudio) + self.assertEqual(with_ollama, alias) + def test_ollama_serves_its_first_non_embedding_model(self): model, served = self._both( roles=["svc-ai-ollama"], @@ -90,15 +123,19 @@ class TestLitellmChatModel(unittest.TestCase): def test_the_model_is_named_exactly_when_one_is_served(self): cases = ( - (["web-app-mattermost"], "", ()), - (["web-app-mattermost"], "sk-test", ()), - (["svc-ai-lmstudio"], "", ()), - (["svc-ai-ollama"], "", ("llama3.2",)), + (["web-app-mattermost"], "", (), ()), + (["web-app-mattermost"], "sk-test", (), ()), + (["svc-ai-lmstudio"], "", (), ()), + (["svc-ai-lmstudio"], "", (), ("qwen2.5:0.5b",)), + (["svc-ai-ollama"], "", ("llama3.2",), ()), ) - for roles, api_key, preload in cases: - with self.subTest(roles=roles, api_key=bool(api_key)): + for roles, api_key, preload, lmstudio in cases: + with self.subTest(roles=roles, api_key=bool(api_key), lmstudio=lmstudio): model, served = self._both( - roles=roles, api_key=api_key, preload_models=preload + roles=roles, + api_key=api_key, + preload_models=preload, + lmstudio_models=lmstudio, ) self.assertEqual(bool(model), served == "True") diff --git a/tests/unit/python/roles/svc-ai-litellm/templates/__init__.py b/tests/unit/python/roles/svc-ai-litellm/templates/__init__.py new file mode 100644 index 0000000000..9f69c70b47 --- /dev/null +++ b/tests/unit/python/roles/svc-ai-litellm/templates/__init__.py @@ -0,0 +1,3 @@ +from pathlib import Path + +PROJECT_ROOT: Path = Path(__file__).resolve().parents[6] diff --git a/tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py b/tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py new file mode 100644 index 0000000000..2c97d52f4c --- /dev/null +++ b/tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py @@ -0,0 +1,127 @@ +"""The gateway config publishes one alias per model, whichever backend serves it. + +``config.yaml.j2`` is where the two local backends meet: Ollama pulls a model +under the alias and LM Studio under its own hub name, and both must surface to +consumers as the same ``model_name``. A wrong accessor or a missing de-duplicate +condition does not fail the render, it publishes a route nothing answers, so the +template is rendered here rather than read. +""" + +from __future__ import annotations + +import unittest + +import yaml +from jinja2 import Environment, StrictUndefined + +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +TEMPLATE = PROJECT_ROOT / "roles/svc-ai-litellm/templates/config.yaml.j2" + +SHARED = {"alias": "qwen2.5:0.5b", "name": "qwen2.5-0.5b-instruct"} +OLLAMA_ONLY = {"alias": "llama3:latest", "name": "llama-3.2-3b-instruct"} +LMSTUDIO_ONLY = {"alias": "mistral:latest", "name": "mistral-7b-instruct-v0.3"} + +OLLAMA_URL = "http://ollama:11434" +LMSTUDIO_URL = "http://lmstudio:1234" + + +def _stub_lookup(ollama_models, lmstudio_models): + def lookup(kind, role, path, *args): + if (kind, path) == ("config", "services.ollama.preload_models"): + return ollama_models + if (kind, path) == ("config", "services.lmstudio.preload_models"): + return lmstudio_models + raise AssertionError(f"unexpected lookup({kind!r}, {role!r}, {path!r})") + + return lookup + + +def render(*, ollama=(), lmstudio=(), api_key=""): + """The rendered config as a parsed mapping. + + Args: + ollama: preload entries svc-ai-ollama declares; empty means not deployed. + lmstudio: preload entries svc-ai-lmstudio declares; empty means not deployed. + api_key: the OpenRouter key, empty when none is configured. + """ + group_names = ["svc-ai-litellm"] + if ollama: + group_names.append("svc-ai-ollama") + if lmstudio: + group_names.append("svc-ai-lmstudio") + env = Environment(undefined=StrictUndefined, autoescape=False) # noqa: S701 - YAML, not markup + rendered = env.from_string(read_text(str(TEMPLATE))).render( + group_names=group_names, + lookup=_stub_lookup(list(ollama), list(lmstudio)), + OLLAMA_BASE_LOCAL_URL=OLLAMA_URL, + LMSTUDIO_BASE_LOCAL_URL=LMSTUDIO_URL, + LITELLM_MAX_OUTPUT_TOKENS=512, + LITELLM_UPSTREAM_TIMEOUT=60, + LITELLM_OPENROUTER_API_KEY=api_key, + ) + return yaml.safe_load( + rendered + ) # nocheck: direct-yaml - parses this test's own render + + +def routes(config) -> dict: + return { + entry["model_name"]: entry["litellm_params"] + for entry in (config.get("model_list") or []) + } + + +class TestOllamaOnly(unittest.TestCase): + def test_every_alias_is_published_against_the_ollama_backend(self) -> None: + published = routes(render(ollama=[SHARED, OLLAMA_ONLY])) + self.assertEqual(set(published), {SHARED["alias"], OLLAMA_ONLY["alias"]}) + self.assertEqual( + published[SHARED["alias"]]["model"], f"ollama/{SHARED['alias']}" + ) + self.assertEqual(published[SHARED["alias"]]["api_base"], OLLAMA_URL) + + +class TestLmstudioOnly(unittest.TestCase): + def test_the_alias_is_published_against_the_lmstudio_name(self) -> None: + published = routes(render(lmstudio=[SHARED, LMSTUDIO_ONLY])) + self.assertEqual(set(published), {SHARED["alias"], LMSTUDIO_ONLY["alias"]}) + self.assertEqual( + published[SHARED["alias"]]["model"], f"openai/{SHARED['name']}" + ) + self.assertEqual(published[SHARED["alias"]]["api_base"], f"{LMSTUDIO_URL}/v1") + + def test_an_alias_reaches_the_same_name_ollama_would_publish(self) -> None: + self.assertEqual( + set(routes(render(lmstudio=[SHARED]))), + set(routes(render(ollama=[SHARED]))), + ) + + +class TestBothBackends(unittest.TestCase): + def test_a_shared_alias_is_published_once_and_routed_to_ollama(self) -> None: + published = routes( + render(ollama=[SHARED, OLLAMA_ONLY], lmstudio=[SHARED, LMSTUDIO_ONLY]) + ) + self.assertEqual( + set(published), + {SHARED["alias"], OLLAMA_ONLY["alias"], LMSTUDIO_ONLY["alias"]}, + ) + self.assertEqual(published[SHARED["alias"]]["api_base"], OLLAMA_URL) + self.assertEqual( + published[LMSTUDIO_ONLY["alias"]]["api_base"], f"{LMSTUDIO_URL}/v1" + ) + + +class TestNoBackend(unittest.TestCase): + def test_the_model_list_is_empty(self) -> None: + self.assertEqual(render()["model_list"], []) + + def test_the_openrouter_key_alone_publishes_its_route(self) -> None: + self.assertEqual(set(routes(render(api_key="sk-test"))), {"openrouter/auto"}) + + +if __name__ == "__main__": + unittest.main() From 328113e7c3a534e51c04a518bc0ac59824105ac9 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 19:23:59 +0200 Subject: [PATCH 012/197] refactor(ai): declare the litellm consumer contract once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourteen roles repeated the same two lines to say they use the gateway: a `shared` flag gated on group_names, and an `enabled` flag conjoining that gate with LITELLM_CHAT_MODEL_SERVED. Both are now single-sourced. `shared` becomes literal true. The flag means the provider is a shared instance rather than one deployed per consumer, and a central gateway is always that; the dynamic form only said "shared if deployed", which `enabled` already decides. The 50 shared pins under `litellm:` leave meta/variants.yml with it. The tree had already noticed: 20 of those 50 lines carried `# nocheck: variants-static-override`, two of them with copy-pasted text about container_backup. This narrows e5de4e085a for one service key rather than reversing it; every other dynamic flag keeps the group_names form. `enabled` becomes `{{ LITELLM_USABLE }}`, defined in group_vars/all/16_ai.yml. That shorthand is only possible because is_explicit_truth learns about it. The resolver reads a dependency out of the flag TEXT โ€” "in group_names" in value โ€” so a constant hides the role id and the provider silently leaves the static closure of every role referencing it. Measured: the old expression resolves True, `{{ LITELLM_USABLE }}` resolves False. GROUP_GATED_CONSTANTS names the constants whose own definition carries the gate, and the docstring states the obligation: a SPOT of this kind that is not listed there drops its provider without a word. The wider fix, accepting any Jinja as dynamic, was measured and rejected: 35 other flags carry a Jinja value without the marker (MCP sidecars, javascript and SSO gates, recaptcha), and they would all enter closures they are not in today. A literal shared flag is no longer dynamic, so test_dynamic_flags wants the `# nocheck: dynamic-flag` its own message documents for legitimate exceptions; where a marker already existed the rule list is extended rather than a second comment added. Behaviour is unchanged where it counts. The closure keeps litellm with the same app counts (wordpress 26, openwebui 23, hermes 23, mediawiki 22, flowise 23), unrelated Jinja still resolves False, and the truth table of the new enabled form matches the old one in all four served/deployed combinations. Three roles resolve without litellm, and `git show HEAD:` proves that predates this change: zammad pins enabled false in variant 0 at line 39, nextcloud at line 97. Six unit tests cover the resolver, including one asserting that every registered constant is actually defined in group_vars, so an entry cannot become a dead promise. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/16_ai.yml | 2 + roles/web-app-discourse/meta/services.yml | 6 +-- roles/web-app-discourse/meta/variants.yml | 2 - roles/web-app-flowise/meta/services.yml | 6 +-- roles/web-app-flowise/meta/variants.yml | 3 -- roles/web-app-hermes/meta/services.yml | 6 +-- roles/web-app-hermes/meta/variants.yml | 3 -- roles/web-app-matrix/meta/services.yml | 5 ++- roles/web-app-matrix/meta/variants.yml | 3 -- roles/web-app-mattermost/meta/services.yml | 5 ++- roles/web-app-mattermost/meta/variants.yml | 3 -- roles/web-app-mediawiki/meta/services.yml | 6 +-- roles/web-app-mediawiki/meta/variants.yml | 3 -- roles/web-app-moodle/meta/services.yml | 5 ++- roles/web-app-moodle/meta/variants.yml | 3 -- roles/web-app-n8n/meta/services.yml | 5 ++- roles/web-app-n8n/meta/variants.yml | 2 - roles/web-app-nextcloud/meta/services.yml | 6 +-- roles/web-app-nextcloud/meta/variants.yml | 14 ------- roles/web-app-openclaw/meta/services.yml | 6 +-- roles/web-app-openclaw/meta/variants.yml | 3 -- roles/web-app-openwebui/meta/services.yml | 6 +-- roles/web-app-openwebui/meta/variants.yml | 3 -- roles/web-app-wordpress/meta/services.yml | 6 +-- roles/web-app-wordpress/meta/variants.yml | 2 - roles/web-app-xwiki/meta/services.yml | 5 ++- roles/web-app-xwiki/meta/variants.yml | 3 -- roles/web-app-zammad/meta/services.yml | 5 ++- roles/web-app-zammad/meta/variants.yml | 3 -- .../applications/services/test_registry.py | 40 +++++++++++++++++++ utils/roles/applications/services/registry.py | 21 ++++++++-- 31 files changed, 102 insertions(+), 89 deletions(-) diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index 63ce93dc2c..7906138287 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -35,6 +35,8 @@ LITELLM_CHAT_MODEL_SERVED: >- | map(attribute='alias') | reject('search', 'embed') | list) | length > 0)) or ((OPENROUTER_API_KEY | string) | length > 0) }} +LITELLM_USABLE: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" + OPENROUTER_API_KEY: "" MCP_DISCOVERY: "{{ lookup('mcp_servers') }}" diff --git a/roles/web-app-discourse/meta/services.yml b/roles/web-app-discourse/meta/services.yml index 162066122a..3fbcfa6ea3 100644 --- a/roles/web-app-discourse/meta/services.yml +++ b/roles/web-app-discourse/meta/services.yml @@ -1,10 +1,10 @@ --- # nocheck: seaweedfs-required โ€” Discourse's aws-sdk-s3 defaults a custom endpoint to virtual-host addressing and exposes no force_path_style option, incompatible with SeaweedFS path-style without a global SeaweedFS virtual-host/wildcard-DNS change affecting all consumers; uploads stay on the local volume -# nocheck: playwright-service-flag - shared gateway consumer; inference wiring is covered by the gateway role +# nocheck: playwright-service-flag, dynamic-flag - shared gateway consumer; inference wiring is covered by the gateway role, and the gateway is a single shared instance litellm: bond: 0.5 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true # nocheck: playwright-service-flag โ€” static-asset provider, no Playwright surface asset: bond: 1 diff --git a/roles/web-app-discourse/meta/variants.yml b/roles/web-app-discourse/meta/variants.yml index 397fa8ca4d..d36f140319 100644 --- a/roles/web-app-discourse/meta/variants.yml +++ b/roles/web-app-discourse/meta/variants.yml @@ -22,7 +22,6 @@ shared: true litellm: enabled: true - shared: true sso: enabled: true shared: true @@ -59,7 +58,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false sso: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true diff --git a/roles/web-app-flowise/meta/services.yml b/roles/web-app-flowise/meta/services.yml index f668766d59..a8d7cb5f01 100644 --- a/roles/web-app-flowise/meta/services.yml +++ b/roles/web-app-flowise/meta/services.yml @@ -28,11 +28,11 @@ sso: ldap: enabled: false shared: false -# nocheck: playwright-service-flag - shared gateway consumer; inference wiring is covered by the gateway role +# nocheck: playwright-service-flag, dynamic-flag - shared gateway consumer; inference wiring is covered by the gateway role, and the gateway is a single shared instance litellm: bond: 1 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true # nocheck: playwright-service-flag - DB engine, covered by role-local integration tests qdrant: bond: 1 diff --git a/roles/web-app-flowise/meta/variants.yml b/roles/web-app-flowise/meta/variants.yml index 2ee01b19b7..6464f2d6b0 100644 --- a/roles/web-app-flowise/meta/variants.yml +++ b/roles/web-app-flowise/meta/variants.yml @@ -28,7 +28,6 @@ shared: true litellm: enabled: true - shared: true qdrant: enabled: true shared: true @@ -68,7 +67,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true qdrant: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -108,7 +106,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true qdrant: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true diff --git a/roles/web-app-hermes/meta/services.yml b/roles/web-app-hermes/meta/services.yml index 883cbff9ff..eeb8442dc1 100644 --- a/roles/web-app-hermes/meta/services.yml +++ b/roles/web-app-hermes/meta/services.yml @@ -60,11 +60,11 @@ logout: enabled: false shared: false -# nocheck: playwright-service-flag +# nocheck: playwright-service-flag, dynamic-flag - the gateway is a single shared instance, never deployed per consumer litellm: bond: 1 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true dashboard: bond: 1 diff --git a/roles/web-app-hermes/meta/variants.yml b/roles/web-app-hermes/meta/variants.yml index ca2f16924d..676b2d63da 100644 --- a/roles/web-app-hermes/meta/variants.yml +++ b/roles/web-app-hermes/meta/variants.yml @@ -10,7 +10,6 @@ shared: true litellm: enabled: true - shared: true dashboard: enabled: true shared: true @@ -34,7 +33,6 @@ shared: true litellm: enabled: false - shared: false # nocheck: variants-static-override variant pins a literal off-switch dashboard: enabled: false shared: false # nocheck: variants-static-override variant pins a literal off-switch @@ -58,7 +56,6 @@ shared: false # nocheck: variants-static-override variant pins a literal off-switch litellm: enabled: false - shared: false # nocheck: variants-static-override variant pins a literal off-switch dashboard: enabled: false shared: false # nocheck: variants-static-override variant pins a literal off-switch diff --git a/roles/web-app-matrix/meta/services.yml b/roles/web-app-matrix/meta/services.yml index fe06542181..77e592c100 100644 --- a/roles/web-app-matrix/meta/services.yml +++ b/roles/web-app-matrix/meta/services.yml @@ -1,8 +1,9 @@ --- +# nocheck: dynamic-flag - the gateway is a single shared instance, never deployed per consumer litellm: bond: 0.5 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true # nocheck: dockerfile-custom sso: bond: 1 diff --git a/roles/web-app-matrix/meta/variants.yml b/roles/web-app-matrix/meta/variants.yml index 13577c808b..59965eb1f1 100644 --- a/roles/web-app-matrix/meta/variants.yml +++ b/roles/web-app-matrix/meta/variants.yml @@ -29,7 +29,6 @@ shared: true litellm: enabled: false - shared: false sso: enabled: true shared: true @@ -73,7 +72,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false sso: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -120,7 +118,6 @@ shared: true litellm: enabled: true - shared: true sso: enabled: true shared: true diff --git a/roles/web-app-mattermost/meta/services.yml b/roles/web-app-mattermost/meta/services.yml index 7964054a8d..c02e2a8a69 100644 --- a/roles/web-app-mattermost/meta/services.yml +++ b/roles/web-app-mattermost/meta/services.yml @@ -113,10 +113,11 @@ container_backup: # nocheck: bond-baseline backup host joins outside the variant bond: 1 enabled: "{{ 'svc-bkp-volume-2-local' in group_names }}" shared: true +# nocheck: dynamic-flag - the gateway is a single shared instance, never deployed per consumer litellm: bond: 0.5 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true # nocheck: playwright-service-flag internal adapter sidecar, covered by the role's MCP spec # nocheck: dockerfile-custom the Dockerfile is owned by svc-ai-mcp-adapter and staged per instance, so every adapter-backed provider shares one audited build # nocheck: dynamic-flag the sidecar tracks this role's own mcp flag, not another role's presence; a group_names expression would decide its exposure from an unrelated deployment diff --git a/roles/web-app-mattermost/meta/variants.yml b/roles/web-app-mattermost/meta/variants.yml index a4d3ad527a..6f53b2a005 100644 --- a/roles/web-app-mattermost/meta/variants.yml +++ b/roles/web-app-mattermost/meta/variants.yml @@ -37,7 +37,6 @@ shared: true litellm: enabled: false - shared: false tor: enabled: true openwebui: @@ -83,7 +82,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false tor: enabled: true openwebui: @@ -130,7 +128,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: true - shared: true tor: enabled: true openwebui: diff --git a/roles/web-app-mediawiki/meta/services.yml b/roles/web-app-mediawiki/meta/services.yml index b1057e1216..8bd87a6e9c 100644 --- a/roles/web-app-mediawiki/meta/services.yml +++ b/roles/web-app-mediawiki/meta/services.yml @@ -1,9 +1,9 @@ --- -# nocheck: playwright-service-flag - shared gateway consumer; inference wiring is covered by the gateway role +# nocheck: playwright-service-flag, dynamic-flag - shared gateway consumer; inference wiring is covered by the gateway role, and the gateway is a single shared instance litellm: bond: 0.5 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true # nocheck: seaweedfs-required โ€” object storage not implemented for this role sso: bond: 1 diff --git a/roles/web-app-mediawiki/meta/variants.yml b/roles/web-app-mediawiki/meta/variants.yml index 03d842033d..bd762b7a90 100644 --- a/roles/web-app-mediawiki/meta/variants.yml +++ b/roles/web-app-mediawiki/meta/variants.yml @@ -26,7 +26,6 @@ shared: true litellm: enabled: true - shared: true sso: enabled: true shared: true @@ -58,7 +57,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false sso: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -91,7 +89,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: true - shared: true sso: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true diff --git a/roles/web-app-moodle/meta/services.yml b/roles/web-app-moodle/meta/services.yml index 4775c187a6..b80661b5db 100644 --- a/roles/web-app-moodle/meta/services.yml +++ b/roles/web-app-moodle/meta/services.yml @@ -1,8 +1,9 @@ --- +# nocheck: dynamic-flag - the gateway is a single shared instance, never deployed per consumer litellm: bond: 0.5 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true # nocheck: seaweedfs-required โ€” object storage not implemented for this role sso: bond: 1 diff --git a/roles/web-app-moodle/meta/variants.yml b/roles/web-app-moodle/meta/variants.yml index 73a44039ba..37f229dd12 100644 --- a/roles/web-app-moodle/meta/variants.yml +++ b/roles/web-app-moodle/meta/variants.yml @@ -38,7 +38,6 @@ shared: true litellm: enabled: false - shared: false tor: enabled: true @@ -83,7 +82,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false tor: enabled: true openwebui: @@ -128,7 +126,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: true - shared: true tor: enabled: true openwebui: diff --git a/roles/web-app-n8n/meta/services.yml b/roles/web-app-n8n/meta/services.yml index f207224900..9fceb2650a 100644 --- a/roles/web-app-n8n/meta/services.yml +++ b/roles/web-app-n8n/meta/services.yml @@ -93,10 +93,11 @@ container_backup: # nocheck: bond-baseline backup host joins outside the variant bond: 1 enabled: "{{ 'svc-bkp-volume-2-local' in group_names }}" shared: true +# nocheck: dynamic-flag - the gateway is a single shared instance, never deployed per consumer litellm: bond: 0.5 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true # nocheck: playwright-service-flag MCP pairing is proven by the role's own MCP spec, not by a per-consumer gate openwebui: bond: 0.5 diff --git a/roles/web-app-n8n/meta/variants.yml b/roles/web-app-n8n/meta/variants.yml index ff9e40548d..a4de279d1d 100644 --- a/roles/web-app-n8n/meta/variants.yml +++ b/roles/web-app-n8n/meta/variants.yml @@ -27,7 +27,6 @@ shared: true litellm: enabled: true - shared: true tor: enabled: true @@ -69,7 +68,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false tor: enabled: true openwebui: diff --git a/roles/web-app-nextcloud/meta/services.yml b/roles/web-app-nextcloud/meta/services.yml index a50f1ac4ed..91bdeb0457 100644 --- a/roles/web-app-nextcloud/meta/services.yml +++ b/roles/web-app-nextcloud/meta/services.yml @@ -335,11 +335,11 @@ openwebui: bond: 0.5 enabled: "{{ 'web-app-openwebui' in group_names }}" shared: "{{ 'web-app-openwebui' in group_names }}" -# nocheck: playwright-service-flag: bridged litellm gateway partner, covered by partner role spec +# nocheck: playwright-service-flag, dynamic-flag - bridged litellm gateway partner, covered by partner role spec, and a single shared instance litellm: bond: 0.5 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true # nocheck: playwright-service-flag: bridged mastodon partner, covered by partner role spec mastodon: bond: 0.5 diff --git a/roles/web-app-nextcloud/meta/variants.yml b/roles/web-app-nextcloud/meta/variants.yml index 29528ef52e..5dad822253 100644 --- a/roles/web-app-nextcloud/meta/variants.yml +++ b/roles/web-app-nextcloud/meta/variants.yml @@ -96,7 +96,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -193,7 +192,6 @@ shared: true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -332,7 +330,6 @@ shared: true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -427,7 +424,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -522,7 +518,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: true shared: true @@ -617,7 +612,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -712,7 +706,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -807,7 +800,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: true - shared: true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -902,7 +894,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -997,7 +988,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -1092,7 +1082,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -1187,7 +1176,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -1282,7 +1270,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -1377,7 +1364,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: true - shared: true mastodon: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true diff --git a/roles/web-app-openclaw/meta/services.yml b/roles/web-app-openclaw/meta/services.yml index ed34538b62..ba79f514df 100644 --- a/roles/web-app-openclaw/meta/services.yml +++ b/roles/web-app-openclaw/meta/services.yml @@ -50,11 +50,11 @@ sso: - "{{ lookup('rbac_group_path', application_id='web-app-openclaw', role='administrator') }}" - "{{ lookup('rbac_group_path', application_id='web-app-openclaw', role='mcp') }}" -# nocheck: playwright-service-flag +# nocheck: playwright-service-flag, dynamic-flag - the gateway is a single shared instance, never deployed per consumer litellm: bond: 1 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true logout: bond: 1 diff --git a/roles/web-app-openclaw/meta/variants.yml b/roles/web-app-openclaw/meta/variants.yml index 355bd2f447..5f0c5ee5b0 100644 --- a/roles/web-app-openclaw/meta/variants.yml +++ b/roles/web-app-openclaw/meta/variants.yml @@ -10,7 +10,6 @@ shared: true litellm: enabled: true - shared: true logout: enabled: true shared: true @@ -37,7 +36,6 @@ shared: true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true logout: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -64,7 +62,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true logout: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true diff --git a/roles/web-app-openwebui/meta/services.yml b/roles/web-app-openwebui/meta/services.yml index 1fe7754ea7..69ea6a7ae6 100644 --- a/roles/web-app-openwebui/meta/services.yml +++ b/roles/web-app-openwebui/meta/services.yml @@ -81,11 +81,11 @@ css: javascript: enabled: false # nocheck: dynamic-flag shared: true -# nocheck: playwright-service-flag โ€” AI backend, infrastructural +# nocheck: playwright-service-flag, dynamic-flag โ€” AI backend, infrastructural, and a single shared instance litellm: bond: 1 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true email: bond: 1 diff --git a/roles/web-app-openwebui/meta/variants.yml b/roles/web-app-openwebui/meta/variants.yml index c47c1396c1..13acca7e3c 100644 --- a/roles/web-app-openwebui/meta/variants.yml +++ b/roles/web-app-openwebui/meta/variants.yml @@ -27,7 +27,6 @@ shared: true litellm: enabled: true - shared: true prometheus: enabled: true shared: true @@ -63,7 +62,6 @@ shared: true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true prometheus: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -102,7 +100,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true prometheus: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true diff --git a/roles/web-app-wordpress/meta/services.yml b/roles/web-app-wordpress/meta/services.yml index 04328dda3a..956a880dbc 100644 --- a/roles/web-app-wordpress/meta/services.yml +++ b/roles/web-app-wordpress/meta/services.yml @@ -1,9 +1,9 @@ --- -# nocheck: playwright-service-flag - shared gateway consumer; inference wiring is covered by the gateway role +# nocheck: playwright-service-flag, dynamic-flag - shared gateway consumer; inference wiring is covered by the gateway role, and the gateway is a single shared instance litellm: bond: 0.5 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true # nocheck: seaweedfs-required, object storage not implemented for this role sso: bond: 1 diff --git a/roles/web-app-wordpress/meta/variants.yml b/roles/web-app-wordpress/meta/variants.yml index 410f2b4110..1416c0955b 100644 --- a/roles/web-app-wordpress/meta/variants.yml +++ b/roles/web-app-wordpress/meta/variants.yml @@ -35,7 +35,6 @@ shared: true litellm: enabled: true - shared: true sso: enabled: true shared: true @@ -79,7 +78,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false sso: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true diff --git a/roles/web-app-xwiki/meta/services.yml b/roles/web-app-xwiki/meta/services.yml index 874eab7b6d..0ef4b47b3a 100644 --- a/roles/web-app-xwiki/meta/services.yml +++ b/roles/web-app-xwiki/meta/services.yml @@ -1,8 +1,9 @@ --- +# nocheck: dynamic-flag - the gateway is a single shared instance, never deployed per consumer litellm: bond: 0.5 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true # nocheck: seaweedfs-required โ€” object storage not implemented for this role sso: bond: 1 diff --git a/roles/web-app-xwiki/meta/variants.yml b/roles/web-app-xwiki/meta/variants.yml index b9265fdd57..97baf06457 100644 --- a/roles/web-app-xwiki/meta/variants.yml +++ b/roles/web-app-xwiki/meta/variants.yml @@ -7,7 +7,6 @@ - services: litellm: enabled: false - shared: false container_backup: enabled: true dashboard: @@ -40,7 +39,6 @@ - services: litellm: enabled: false - shared: false container_backup: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true @@ -74,7 +72,6 @@ - services: # nocheck: variants-auth-isolation litellm: enabled: true - shared: true container_backup: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true diff --git a/roles/web-app-zammad/meta/services.yml b/roles/web-app-zammad/meta/services.yml index 254bc7e88d..35ffa8e682 100644 --- a/roles/web-app-zammad/meta/services.yml +++ b/roles/web-app-zammad/meta/services.yml @@ -1,8 +1,9 @@ --- +# nocheck: dynamic-flag - the gateway is a single shared instance, never deployed per consumer litellm: bond: 0.5 - enabled: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" - shared: "{{ 'svc-ai-litellm' in group_names }}" + enabled: "{{ LITELLM_USABLE }}" + shared: true # nocheck: seaweedfs-required - object storage not implemented for this role sso: bond: 1 diff --git a/roles/web-app-zammad/meta/variants.yml b/roles/web-app-zammad/meta/variants.yml index b432f5a7d4..38cb4ddfb5 100644 --- a/roles/web-app-zammad/meta/variants.yml +++ b/roles/web-app-zammad/meta/variants.yml @@ -38,7 +38,6 @@ shared: true litellm: enabled: false - shared: false tor: enabled: true openwebui: @@ -86,7 +85,6 @@ shared: false # nocheck: variants-static-override backup shared stays literally true litellm: enabled: false - shared: false tor: enabled: true @@ -117,7 +115,6 @@ shared: true litellm: enabled: true - shared: true logout: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true diff --git a/tests/unit/python/utils/roles/applications/services/test_registry.py b/tests/unit/python/utils/roles/applications/services/test_registry.py index ab7de60a13..725eee68c5 100644 --- a/tests/unit/python/utils/roles/applications/services/test_registry.py +++ b/tests/unit/python/utils/roles/applications/services/test_registry.py @@ -4,12 +4,15 @@ import tempfile import unittest from pathlib import Path +from utils.cache.files import PROJECT_ROOT from utils.cache.yaml import dump_yaml_str from utils.roles.applications.services.registry import ( + GROUP_GATED_CONSTANTS, ServiceRegistryError, build_service_registry_from_applications, detect_service_bucket, detect_service_channel, + is_explicit_truth, ordered_primary_service_entries, resolve_service_dependency_roles_from_config, ) @@ -335,5 +338,42 @@ class TestServiceRegistryOrdering(unittest.TestCase): ordered_primary_service_entries(registry, roles_dir) +class TestExplicitTruth(unittest.TestCase): + """A flag may resolve a dependency through a group-gated constant. + + The substring rule reads the role id out of the flag itself, so a SPOT + constant hides it. Roles referencing one keep their provider only while + the constant is registered. + """ + + def test_a_literal_true_resolves(self): + self.assertTrue(is_explicit_truth(True)) + + def test_the_in_group_names_form_resolves(self): + self.assertTrue(is_explicit_truth("{{ 'web-app-keycloak' in group_names }}")) + + def test_every_registered_constant_resolves(self): + for constant in GROUP_GATED_CONSTANTS: + with self.subTest(constant=constant): + self.assertTrue(is_explicit_truth("{{ " + constant + " }}")) + + def test_an_unregistered_jinja_expression_does_not_resolve(self): + self.assertFalse( + is_explicit_truth("{{ lookup('config', 'web-app-gitea', 'mcp.enabled') }}") + ) + + def test_a_literal_false_does_not_resolve(self): + self.assertFalse(is_explicit_truth(False)) + + def test_every_registered_constant_is_defined_in_group_vars(self): + group_vars = PROJECT_ROOT / "group_vars" / "all" + defined = "".join( + path.read_text(encoding="utf-8") for path in group_vars.glob("*.yml") + ) + for constant in GROUP_GATED_CONSTANTS: + with self.subTest(constant=constant): + self.assertIn(f"{constant}:", defined) + + if __name__ == "__main__": unittest.main() diff --git a/utils/roles/applications/services/registry.py b/utils/roles/applications/services/registry.py index 561ce13652..484bf2ada3 100644 --- a/utils/roles/applications/services/registry.py +++ b/utils/roles/applications/services/registry.py @@ -28,10 +28,21 @@ def _normalized_name(value: Any) -> str: return value.strip() +GROUP_GATED_CONSTANTS: tuple[str, ...] = ("LITELLM_USABLE",) +"""Group_vars constants whose own definition carries an ``in group_names`` +gate, so a flag referencing one is dynamic even though its own text is not. + +A constant hides the role id the substring rule looks for, so every SPOT of +this kind MUST be listed here or the roles referencing it silently drop their +provider from the static closure. +""" + + def is_explicit_truth(value: Any) -> bool: """A services. flag (``enabled`` / ``shared``) MAY be true if it - is the literal Python ``True`` OR any string containing the substring - ``in group_names``. Negated and compound predicates + is the literal Python ``True``, any string containing the substring + ``in group_names``, or a string referencing a `GROUP_GATED_CONSTANTS` + entry. Negated and compound predicates (``"{{ 'X' not in group_names }}"``) match on purpose: they are true on some node shape, and the static closure is the co-deploy superset. Mirrors the substring test at @@ -47,7 +58,11 @@ def is_explicit_truth(value: Any) -> bool: """ if value is True: return True - return bool(isinstance(value, str) and "in group_names" in value) + if not isinstance(value, str): + return False + return "in group_names" in value or any( + constant in value for constant in GROUP_GATED_CONSTANTS + ) def detect_service_channel(role_name: str) -> str: From 4d1c8c52f5dcb6e35e5e93293ffaf9a98f89bc92 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 19:25:41 +0200 Subject: [PATCH 013/197] feat(litellm): give LM Studio a variant that deploys it alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Variant 0 pinned both local backends on, so LITELLM_CHAT_MODEL always resolved to an Ollama model and every completion went there. The LM Studio route was published and never travelled. A third variant turns Ollama off and LM Studio on, which is the only shape that exercises it, and variant 0 pins LM Studio literally off so a WordPress round stops co-deploying a backend it never prompts: 27 apps become 26. bond drops to 0.5 and that is what makes the variant legal. test_order's _direct_deps filters services on bond >= 1.0 before building a closure, so at 0.5 LM Studio leaves that computation and the new variant no longer violates "variant 0 is the maximal deploy footprint". The deploy planner does not filter on bond โ€” resolve_service_dependency_roles_from_config reads enabled and shared only โ€” so the variant still deploys it. test_bond_baseline takes its bond < 1 branch, which asks for any variant pinning the flag true rather than variant 0 specifically, and variant 2 supplies it. No suppression and no guard change. bond < 1 states that the provider is a separate-host partner rather than a same-host service. Reaching the gateway through LMSTUDIO_BASE_LOCAL_URL, a container-network hostname, is consistent with that. Measured against the planner: variant 0 resolves 21 apps without LM Studio, variant 1 stays at 1, variant 2 resolves 8 apps with LM Studio and without Ollama, and a WordPress round 0 drops from 27 to 26. The variant is reachable only when svc-ai-litellm is swept as a primary row and pinned to it: legacy_resolver clamps a non-primary role to the round index, and a two-round WordPress job never reaches index 2. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-litellm/meta/services.yml | 2 +- roles/svc-ai-litellm/meta/variants.yml | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/roles/svc-ai-litellm/meta/services.yml b/roles/svc-ai-litellm/meta/services.yml index 39c3430c6e..5d7177968c 100644 --- a/roles/svc-ai-litellm/meta/services.yml +++ b/roles/svc-ai-litellm/meta/services.yml @@ -42,6 +42,6 @@ ollama: # nocheck: playwright-service-flag, dynamic-flag lmstudio: - bond: 1 + bond: 0.5 enabled: "{{ 'svc-ai-lmstudio' in group_names }}" shared: "{{ 'svc-ai-lmstudio' in group_names }}" diff --git a/roles/svc-ai-litellm/meta/variants.yml b/roles/svc-ai-litellm/meta/variants.yml index be12f8165f..8ae1ddd8aa 100644 --- a/roles/svc-ai-litellm/meta/variants.yml +++ b/roles/svc-ai-litellm/meta/variants.yml @@ -7,8 +7,8 @@ enabled: true shared: true lmstudio: - enabled: true - shared: true + enabled: false + shared: false postgres: shared: true @@ -21,3 +21,14 @@ lmstudio: enabled: false shared: false # nocheck: variants-static-override backup shared stays literally true + +- services: + ollama: + enabled: false + shared: false + lmstudio: + bond: 0.5 + enabled: true + shared: true + postgres: + shared: true From be01db963f1f0b7e1d95ae4d362272a9bd5cad62 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 19:26:36 +0200 Subject: [PATCH 014/197] fix(tests): read group_vars through the cache in the registry guard 328113e7c added a guard that reads every group_vars/all file to confirm each GROUP_GATED_CONSTANTS entry is really defined, and read them with path.read_text(). test_no_uncached_file_reads rejects that under tests/: the cached read_text shares one decoded read per file across the whole session, and these files are read by several other tests already. The nocheck escape does not apply. It covers reads outside the project tree, and group_vars/all is inside it. What the guard asserts is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- .../utils/roles/applications/services/test_registry.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/python/utils/roles/applications/services/test_registry.py b/tests/unit/python/utils/roles/applications/services/test_registry.py index 725eee68c5..f27085ea7c 100644 --- a/tests/unit/python/utils/roles/applications/services/test_registry.py +++ b/tests/unit/python/utils/roles/applications/services/test_registry.py @@ -4,7 +4,7 @@ import tempfile import unittest from pathlib import Path -from utils.cache.files import PROJECT_ROOT +from utils.cache.files import PROJECT_ROOT, read_text from utils.cache.yaml import dump_yaml_str from utils.roles.applications.services.registry import ( GROUP_GATED_CONSTANTS, @@ -367,9 +367,7 @@ class TestExplicitTruth(unittest.TestCase): def test_every_registered_constant_is_defined_in_group_vars(self): group_vars = PROJECT_ROOT / "group_vars" / "all" - defined = "".join( - path.read_text(encoding="utf-8") for path in group_vars.glob("*.yml") - ) + defined = "".join(read_text(str(path)) for path in group_vars.glob("*.yml")) for constant in GROUP_GATED_CONSTANTS: with self.subTest(constant=constant): self.assertIn(f"{constant}:", defined) From abd6a24b73a5ee970de6df26694df1662f0b3d3a Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 19:27:53 +0200 Subject: [PATCH 015/197] test(litellm): assert the gateway routes to the backends it actually has MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway publishing a model proves nothing about a backend answering it: LM Studio shipped a published lmstudio/default route for as long as it had no model to load. This CLI test closes that gap by asking the running gateway what it serves and comparing it against what the deployment enabled. templates/test.env.j2 is both the input and the entry point โ€” discover_cli_roles finds a role by that file and filters on group_names, so the test runs only where the gateway is deployed. meta/tests.yml names the container service key so run_one.yml resolves the node holding it in swarm, and allows 600s for a cold model load. probe.py runs inside the gateway container, where the API answers on loopback and the backends resolve by name, and uses the standard library only because the image ships the litellm app rather than a test toolchain. It asserts that every declared model is served, that the LM-Studio-only aliases are present exactly when that backend is deployed, that no Ollama-backed model is published without Ollama, that LITELLM_CHAT_MODEL is routable, that an empty deployment publishes nothing, and that a real completion comes back non-empty. The fourth assertion is the one worth having: LITELLM_CHAT_MODEL is built from lookup('deployment').inventory while the config is built from this host's group_names. Two sources for one fact, and this is where they are made to agree. The alias list is rendered whether or not LM Studio is deployed. Gating it would empty the set in exactly the case the negative direction needs, turning "a route must not stand without its backend" into a vacuous check. The verdict logic sits in a pure evaluate() so it is exercised without mocks: four deployment shapes, each with its positive and negative cases, including a both-backends case that asserts a shared alias proves nothing about LM Studio and only the exclusive one counts. files/test/.nocheck is required rather than convenient. role-files-layout sorts files/ by language while test-e2e-cli stages files/test/ under that fixed path; the other 29 CLI test directories carry the same marker. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-litellm/files/test/.nocheck | 1 + roles/svc-ai-litellm/files/test/probe.py | 187 ++++++++++++++++++ roles/svc-ai-litellm/files/test/test.sh | 78 ++++++++ roles/svc-ai-litellm/meta/tests.yml | 4 + roles/svc-ai-litellm/templates/test.env.j2 | 15 ++ .../files/test_probe_routing.py | 173 ++++++++++++++++ 6 files changed, 458 insertions(+) create mode 100644 roles/svc-ai-litellm/files/test/.nocheck create mode 100644 roles/svc-ai-litellm/files/test/probe.py create mode 100644 roles/svc-ai-litellm/files/test/test.sh create mode 100644 roles/svc-ai-litellm/meta/tests.yml create mode 100644 roles/svc-ai-litellm/templates/test.env.j2 create mode 100644 tests/unit/python/roles/svc-ai-litellm/files/test_probe_routing.py diff --git a/roles/svc-ai-litellm/files/test/.nocheck b/roles/svc-ai-litellm/files/test/.nocheck new file mode 100644 index 0000000000..2b9d82d840 --- /dev/null +++ b/roles/svc-ai-litellm/files/test/.nocheck @@ -0,0 +1 @@ +# nocheck: role-files-layout - Role test harness with test.sh as its entry point; test-e2e-cli stages files/test/ by that fixed path. diff --git a/roles/svc-ai-litellm/files/test/probe.py b/roles/svc-ai-litellm/files/test/probe.py new file mode 100644 index 0000000000..551d61bfbb --- /dev/null +++ b/roles/svc-ai-litellm/files/test/probe.py @@ -0,0 +1,187 @@ +"""Assert the gateway routes to exactly the backends this deployment enabled. + +Runs inside the litellm container, where the gateway answers on loopback and +the backend containers resolve by name. Standard library only: the image ships +the litellm app, not a test toolchain. + +Env (rendered into test.env from templates/test.env.j2): + PORT gateway http port inside the container + MASTER_KEY gateway master key + CHAT_MODEL the model every consumer asks for (LITELLM_CHAT_MODEL) + CHAT_MODEL_SERVED true|false, whether a backend can answer it + EXPECTED_MODELS JSON list the config template published + LMSTUDIO_ALIASES JSON list of aliases only LM Studio provides, rendered + whether or not the backend is deployed so the absent + case stays assertable + OLLAMA_ENABLED true|false + LMSTUDIO_ENABLED true|false + RETRIES completion attempts (default 10) + SLEEP_SECONDS wait between attempts (default 6) +""" + +from __future__ import annotations + +import json +import os +import sys +import time +import urllib.error +import urllib.request + +OPENROUTER_MODEL = "openrouter/auto" + + +def evaluate( + *, + served: set[str], + expected: list[str], + chat_model: str, + chat_model_served: bool, + ollama_enabled: bool, + lmstudio_enabled: bool, + lmstudio_aliases: list[str], +) -> list[str]: + """Every routing violation the published model list shows, as messages. + + Args: + served: model ids the gateway answers ``/v1/models`` with. + expected: model ids the rendered config declared. + chat_model: the id consumers are configured to ask for. + chat_model_served: whether any backend can answer *chat_model*. + ollama_enabled: svc-ai-ollama is deployed on the gateway's host. + lmstudio_enabled: svc-ai-lmstudio is deployed on the gateway's host. + lmstudio_aliases: aliases only LM Studio provides, whatever is deployed. + + Returns: + One message per violation; empty when the routing is consistent. + """ + failures: list[str] = [] + + undelivered = sorted(set(expected) - served) + if undelivered: + failures.append( + f"the config declares {undelivered} but the gateway does not serve " + f"them; the rendered config.yaml and the running gateway disagree" + ) + + exclusive = set(lmstudio_aliases) + if lmstudio_enabled and not exclusive <= served: + failures.append( + f"svc-ai-lmstudio is deployed but the gateway does not serve " + f"{sorted(exclusive - served)}; its models were declared and not routed" + ) + if not lmstudio_enabled and exclusive & served: + failures.append( + f"svc-ai-lmstudio is not deployed yet the gateway publishes " + f"{sorted(exclusive & served)}; a route stands without its backend" + ) + + unbacked = sorted(served - exclusive - {OPENROUTER_MODEL}) + if not ollama_enabled and unbacked: + failures.append( + f"svc-ai-ollama is not deployed yet the gateway publishes {unbacked}" + ) + + if not chat_model_served: + if served: + failures.append( + f"no backend is deployed but the gateway publishes {sorted(served)}" + ) + return failures + + if chat_model not in served: + failures.append( + f"consumers are configured for '{chat_model}' (LITELLM_CHAT_MODEL, " + f"built from the deployment inventory) but the gateway serves " + f"{sorted(served)} (built from this host's group_names); the two " + f"sources disagree" + ) + + return failures + + +def _call(base: str, key: str, path: str, payload: dict | None = None): + body = None if payload is None else json.dumps(payload).encode() + request = urllib.request.Request( # noqa: S310 - literal http:// to loopback + base + path, + data=body, + headers={ + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen( # noqa: S310 - literal http:// to loopback + request, timeout=120 + ) as response: + return json.loads(response.read().decode() or "{}") + + +def _completion(base: str, key: str, model: str, retries: int, pause: float) -> str: + last_error = "no attempt ran" + for attempt in range(1, retries + 1): + try: + completion = _call( + base, + key, + "/v1/chat/completions", + { + "model": model, + "messages": [ + {"role": "user", "content": "Reply with the word pong."} + ], + "max_tokens": 16, + }, + ) + choices = completion.get("choices") or [{}] + answer = (choices[0].get("message", {}).get("content") or "").strip() + if answer: + return answer + last_error = "empty content" + except (urllib.error.URLError, TimeoutError, ValueError) as exc: + last_error = f"{type(exc).__name__}: {exc}" + if attempt < retries: + time.sleep(pause) + raise RuntimeError(last_error) + + +def main() -> int: + key = os.environ["MASTER_KEY"] + base = f"http://127.0.0.1:{os.environ['PORT']}" + chat_model = os.environ["CHAT_MODEL"].strip() + chat_model_served = os.environ["CHAT_MODEL_SERVED"] == "true" + retries = int(os.environ.get("RETRIES", "10")) + pause = float(os.environ.get("SLEEP_SECONDS", "6")) + + listing = _call(base, key, "/v1/models") + served = {entry.get("id") for entry in listing.get("data", [])} + print(f"[INFO] gateway answered /v1/models, serving {sorted(served)}") + + failures = evaluate( + served=served, + expected=json.loads(os.environ["EXPECTED_MODELS"]), + chat_model=chat_model, + chat_model_served=chat_model_served, + ollama_enabled=os.environ["OLLAMA_ENABLED"] == "true", + lmstudio_enabled=os.environ["LMSTUDIO_ENABLED"] == "true", + lmstudio_aliases=json.loads(os.environ["LMSTUDIO_ALIASES"]), + ) + + if chat_model_served and not failures: + try: + answer = _completion(base, key, chat_model, retries, pause) + print(f"[OK] '{chat_model}' answered: {answer[:60]}") + except RuntimeError as exc: + failures.append( + f"'{chat_model}' is published but answered nothing after " + f"{retries} attempts; last failure: {exc}" + ) + + for failure in failures: + print(f"[FAIL] {failure}", file=sys.stderr) + if not failures: + print("[OK] the published routes match the deployed backends") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/roles/svc-ai-litellm/files/test/test.sh b/roles/svc-ai-litellm/files/test/test.sh new file mode 100644 index 0000000000..1f733cb4e9 --- /dev/null +++ b/roles/svc-ai-litellm/files/test/test.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# +# svc-ai-litellm routing test. Runs probe.py inside the gateway container, where +# the API answers on loopback and the backends resolve by container name, and +# asserts the published model list matches the backends this deployment enabled. +# +# Env (rendered into test.env from templates/test.env.j2): +# LITELLM_CONTAINER resolved gateway container (CLI_LOCAL_CID) +# LITELLM_PORT gateway http port inside the container +# LITELLM_MASTER_KEY_B64 base64 master key (avoids shell-quoting issues) +# LITELLM_CHAT_MODEL the model every consumer asks for +# LITELLM_CHAT_MODEL_SERVED true|false +# LITELLM_EXPECTED_MODELS JSON list the config template published +# LITELLM_LMSTUDIO_ALIASES JSON list of aliases only LM Studio provides +# LITELLM_OLLAMA_ENABLED true|false +# LITELLM_LMSTUDIO_ENABLED true|false +# READY_RETRIES gateway readiness attempts (default 30) +# READY_SLEEP_SECONDS wait between attempts (default 5) + +set -uo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +READY_RETRIES="${READY_RETRIES:-30}" +READY_SLEEP_SECONDS="${READY_SLEEP_SECONDS:-5}" + +MASTER_KEY="$(printf '%s' "${LITELLM_MASTER_KEY_B64}" | base64 -d 2>/dev/null)" +[ -n "${MASTER_KEY}" ] || { + echo "[FATAL] LITELLM_MASTER_KEY_B64 missing or undecodable" >&2 + exit 2 +} +[ -n "${LITELLM_CONTAINER}" ] || { + echo "[FATAL] LITELLM_CONTAINER unset; the gateway container was not resolved" >&2 + exit 2 +} + +# The image ships the litellm app, so a python interpreter is present, but the +# upstream tag has changed which of the two names is on PATH. +interpreter() { + # nocheck: container-exec-resolver address resolved by the caller and passed in + if container exec -i "${LITELLM_CONTAINER}" python3 -c '' 2>/dev/null; then + echo python3 + else + echo python + fi +} + +probe() { + # nocheck: container-exec-resolver address resolved by the caller and passed in + container exec -i \ + -e "PORT=${LITELLM_PORT}" \ + -e "MASTER_KEY=${MASTER_KEY}" \ + -e "CHAT_MODEL=${LITELLM_CHAT_MODEL}" \ + -e "CHAT_MODEL_SERVED=${LITELLM_CHAT_MODEL_SERVED}" \ + -e "EXPECTED_MODELS=${LITELLM_EXPECTED_MODELS}" \ + -e "LMSTUDIO_ALIASES=${LITELLM_LMSTUDIO_ALIASES}" \ + -e "OLLAMA_ENABLED=${LITELLM_OLLAMA_ENABLED}" \ + -e "LMSTUDIO_ENABLED=${LITELLM_LMSTUDIO_ENABLED}" \ + "${LITELLM_CONTAINER}" "$(interpreter)" - <"${here}/probe.py" +} + +attempt=1 +while [ "${attempt}" -le "${READY_RETRIES}" ]; do + output="$(probe 2>&1)" + rc=$? + case "${output}" in + *"gateway answered /v1/models"*) + echo "${output}" + exit "${rc}" + ;; + esac + attempt=$((attempt + 1)) + [ "${attempt}" -le "${READY_RETRIES}" ] && sleep "${READY_SLEEP_SECONDS}" +done + +echo "[FAIL] the gateway never answered /v1/models after ${READY_RETRIES} attempts" >&2 +echo "${output}" >&2 +exit 1 diff --git a/roles/svc-ai-litellm/meta/tests.yml b/roles/svc-ai-litellm/meta/tests.yml new file mode 100644 index 0000000000..89594be937 --- /dev/null +++ b/roles/svc-ai-litellm/meta/tests.yml @@ -0,0 +1,4 @@ +--- +cli: + timeout: 600 + container_service_key: litellm diff --git a/roles/svc-ai-litellm/templates/test.env.j2 b/roles/svc-ai-litellm/templates/test.env.j2 new file mode 100644 index 0000000000..fe917bb9db --- /dev/null +++ b/roles/svc-ai-litellm/templates/test.env.j2 @@ -0,0 +1,15 @@ +{%- set _ollama_models = (lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') | map(attribute='alias') | list) if 'svc-ai-ollama' in group_names else [] -%} +{%- set _lmstudio_declared = lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') | map(attribute='alias') | list -%} +{%- set _lmstudio_only = _lmstudio_declared | reject('in', _ollama_models) | list -%} +{%- set _expected = _ollama_models + + (_lmstudio_only if 'svc-ai-lmstudio' in group_names else []) + + (['openrouter/auto'] if LITELLM_OPENROUTER_API_KEY else []) -%} +LITELLM_CONTAINER={{ CLI_LOCAL_CID }} +LITELLM_PORT={{ LITELLM_PORT }} +LITELLM_MASTER_KEY_B64={{ LITELLM_MASTER_KEY | b64encode }} +LITELLM_CHAT_MODEL={{ LITELLM_CHAT_MODEL }} +LITELLM_CHAT_MODEL_SERVED={{ LITELLM_CHAT_MODEL_SERVED | bool | string | lower }} +LITELLM_OLLAMA_ENABLED={{ ('svc-ai-ollama' in group_names) | string | lower }} +LITELLM_LMSTUDIO_ENABLED={{ ('svc-ai-lmstudio' in group_names) | string | lower }} +LITELLM_LMSTUDIO_ALIASES='{{ _lmstudio_only | to_json }}' +LITELLM_EXPECTED_MODELS='{{ _expected | to_json }}' diff --git a/tests/unit/python/roles/svc-ai-litellm/files/test_probe_routing.py b/tests/unit/python/roles/svc-ai-litellm/files/test_probe_routing.py new file mode 100644 index 0000000000..f16148508b --- /dev/null +++ b/tests/unit/python/roles/svc-ai-litellm/files/test_probe_routing.py @@ -0,0 +1,173 @@ +"""Unit tests for the svc-ai-litellm routing probe's verdict logic.""" + +from __future__ import annotations + +import importlib.util +import unittest + +from . import PROJECT_ROOT + +MODULE_PATH = PROJECT_ROOT / "roles/svc-ai-litellm/files/test/probe.py" + +spec = importlib.util.spec_from_file_location("litellm_probe", MODULE_PATH) +probe = importlib.util.module_from_spec(spec) +spec.loader.exec_module(probe) + +ALIAS = "qwen2.5:0.5b" +LMSTUDIO_ONLY = "mistral:latest" + + +def verdict(**overrides) -> list[str]: + kwargs = { + "served": set(), + "expected": [], + "chat_model": "", + "chat_model_served": False, + "ollama_enabled": False, + "lmstudio_enabled": False, + "lmstudio_aliases": [LMSTUDIO_ONLY], + } + kwargs.update(overrides) + return probe.evaluate(**kwargs) + + +class TestOllamaOnlyDeployment(unittest.TestCase): + """svc-ai-ollama deployed, svc-ai-lmstudio not โ€” variant 0.""" + + def test_a_consistent_gateway_passes(self) -> None: + self.assertEqual( + verdict( + served={ALIAS}, + expected=[ALIAS], + chat_model=ALIAS, + chat_model_served=True, + ollama_enabled=True, + ), + [], + ) + + def test_an_lmstudio_alias_without_its_backend_fails(self) -> None: + failures = verdict( + served={ALIAS, LMSTUDIO_ONLY}, + expected=[ALIAS], + chat_model=ALIAS, + chat_model_served=True, + ollama_enabled=True, + ) + self.assertEqual(len(failures), 1) + self.assertIn("svc-ai-lmstudio is not deployed", failures[0]) + + def test_a_declared_model_the_gateway_dropped_fails(self) -> None: + failures = verdict( + served=set(), + expected=[ALIAS], + chat_model=ALIAS, + chat_model_served=True, + ollama_enabled=True, + ) + self.assertTrue( + any("does not serve" in failure for failure in failures), failures + ) + + def test_the_model_consumers_ask_for_must_be_routable(self) -> None: + failures = verdict( + served={ALIAS}, + expected=[ALIAS], + chat_model="llama3:latest", + chat_model_served=True, + ollama_enabled=True, + ) + self.assertEqual(len(failures), 1) + self.assertIn("llama3:latest", failures[0]) + + +class TestLmstudioOnlyDeployment(unittest.TestCase): + """svc-ai-lmstudio deployed, svc-ai-ollama not โ€” variant 2.""" + + def test_the_same_alias_ollama_would_serve_passes(self) -> None: + self.assertEqual( + verdict( + served={ALIAS}, + expected=[ALIAS], + chat_model=ALIAS, + chat_model_served=True, + lmstudio_enabled=True, + lmstudio_aliases=[ALIAS], + ), + [], + ) + + def test_a_declared_alias_the_gateway_never_routed_fails(self) -> None: + failures = verdict( + served=set(), + expected=[ALIAS], + chat_model=ALIAS, + chat_model_served=True, + lmstudio_enabled=True, + lmstudio_aliases=[ALIAS], + ) + self.assertTrue( + any("declared and not routed" in failure for failure in failures), failures + ) + + def test_an_ollama_model_without_its_backend_fails(self) -> None: + failures = verdict( + served={ALIAS, "llama3:latest"}, + expected=[ALIAS], + chat_model=ALIAS, + chat_model_served=True, + lmstudio_enabled=True, + lmstudio_aliases=[ALIAS], + ) + self.assertTrue( + any("svc-ai-ollama is not deployed" in failure for failure in failures), + failures, + ) + + +class TestBothBackendsDeployment(unittest.TestCase): + """Variant 0 plus lmstudio: a shared alias proves nothing about lmstudio.""" + + def test_only_the_exclusive_alias_is_demanded(self) -> None: + self.assertEqual( + verdict( + served={ALIAS, LMSTUDIO_ONLY}, + expected=[ALIAS, LMSTUDIO_ONLY], + chat_model=ALIAS, + chat_model_served=True, + ollama_enabled=True, + lmstudio_enabled=True, + ), + [], + ) + + def test_a_missing_exclusive_alias_still_fails(self) -> None: + failures = verdict( + served={ALIAS}, + expected=[ALIAS], + chat_model=ALIAS, + chat_model_served=True, + ollama_enabled=True, + lmstudio_enabled=True, + ) + self.assertTrue( + any("declared and not routed" in failure for failure in failures), failures + ) + + +class TestNoBackendDeployment(unittest.TestCase): + def test_an_empty_gateway_passes(self) -> None: + self.assertEqual(verdict(), []) + + def test_any_published_model_fails(self) -> None: + failures = verdict(served={probe.OPENROUTER_MODEL}) + self.assertTrue( + any("no backend is deployed" in failure for failure in failures), failures + ) + + def test_the_chat_model_is_not_demanded_when_nothing_serves_it(self) -> None: + self.assertEqual(verdict(chat_model=ALIAS), []) + + +if __name__ == "__main__": + unittest.main() From 786899700f09f30023d19d60a26ebc46ff7d9d7d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 19:57:21 +0200 Subject: [PATCH 016/197] feat(litellm): serve OpenAI, Anthropic and OpenRouter from declared keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway could reach exactly one remote provider, through a bare OPENROUTER_API_KEY group_var with an empty-string default and an `{% if %}` around a single hardcoded model entry. OpenAI and Anthropic had no path at all. All three are now declared in roles/svc-ai-litellm/meta/secrets.yml, which mapping.py already calls the application config schema, so a key lands at applications.svc-ai-litellm.secrets.credentials._api_key like every other credential. Each carries the `type` and `regex` its value must satisfy and defaults to the central API..api_key block, which keeps one operator-facing place to fill a provider in while the role owns the shape. type and regex had to become real to be worth declaring. 192 `validation:` blocks sit in the role schemas today and not one line of Python enforces them, so a `regex` would have joined a pile of decoration. validate_supplied_value now runs on every path a value arrives from outside โ€” the --set override, a value already in the inventory, and the value standing next to a `default:`. A generated value stays exempt because the algorithm that produced it is named by the same schema and generate_value takes no schema argument. min_length is deliberately left unenforced: the generators ignore it, so holding them to it would fail where nothing is wrong. The pattern is written for the paste, not for the vendor. OpenAI moved issuance from sk- to sk-proj- without notice and Anthropic issues sk-ant-api03-; a schema pinning a prefix would reject a valid key and abort inventory creation. `\S{20,}` accepts every form those three issue today and one nobody has announced, and rejects a trailing newline, an embedded space, a truncated copy and a . A key pasted into the wrong provider's slot is caught at deploy instead, where the route it publishes answers nothing. The gateway iterates one LITELLM_REMOTE_BACKENDS list instead of three `{% if %}` blocks; config.yaml.j2 and env.j2 read the same list and selectattr('key') decides. LITELLM_CHAT_MODEL falls back to the first configured remote alias rather than to a hardcoded openrouter/auto. For CI the keys travel as INFINITO_API_{OPENAI,ANTHROPIC,OPENROUTER}_KEY through default.env, the passthrough allowlist, the infinito service in compose.yml, the workflow env block and the development inventory. The compose leg is not optional: the env lint rejected the change without it, because a key the inventory reads but compose does not forward resolves to empty inside the container with no error. The workflow reads `secrets.X || vars.X` so a variable works as asked while a secret wins, which matters because a variable is not masked and the development inventory sets MASK_CREDENTIALS_IN_LOGS: false. An unset key is not an error anywhere: the provider's routes stay unpublished and the CLI test asserts their absence rather than skipping. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/call-test-deploy.yml | 3 + compose.yml | 3 + default.env | 9 ++ .../design/role/services/layout.md | 54 ++++++++++++ .../tools/github/actions/configuration.md | 25 ++++++ .../tools/github/actions/secrets.md | 23 +++++ group_vars/all/16_ai.yml | 12 +-- group_vars/all/18_api.yml | 6 ++ inventories/development/default.yml | 8 ++ roles/svc-ai-litellm/README.md | 5 +- roles/svc-ai-litellm/files/test/probe.py | 9 +- roles/svc-ai-litellm/files/test/test.sh | 2 + roles/svc-ai-litellm/meta/secrets.yml | 16 ++++ roles/svc-ai-litellm/templates/config.yaml.j2 | 15 ++-- roles/svc-ai-litellm/templates/env.j2 | 6 +- roles/svc-ai-litellm/templates/test.env.j2 | 6 +- roles/svc-ai-litellm/vars/main.yml | 17 +++- .../tasks/utils/ai_gateway.yml | 2 +- .../group_vars/test_litellm_chat_model.py | 2 +- .../files/test_probe_routing.py | 45 +++++++++- .../templates/test_config_model_list.py | 49 +++++++++-- .../utils/manager/test_inventory_manager.py | 84 ++++++++++++++++++- utils/env/handlers/passthrough.py | 3 + utils/manager/inventory.py | 67 ++++++++++++++- 24 files changed, 438 insertions(+), 33 deletions(-) diff --git a/.github/workflows/call-test-deploy.yml b/.github/workflows/call-test-deploy.yml index 82de4d6579..c3ce5cd652 100644 --- a/.github/workflows/call-test-deploy.yml +++ b/.github/workflows/call-test-deploy.yml @@ -125,6 +125,9 @@ env: INFINITO_IMAGE_TAG: ci-${{ github.sha }} INFINITO_GHCR_MIRROR_PREFIX: mirror INFINITO_PLAYWRIGHT_KEEP: ${{ vars.PLAYWRIGHT_KEEP_ALL }} + INFINITO_API_OPENAI_KEY: ${{ secrets.API_OPENAI_KEY || vars.API_OPENAI_KEY }} + INFINITO_API_ANTHROPIC_KEY: ${{ secrets.API_ANTHROPIC_KEY || vars.API_ANTHROPIC_KEY }} + INFINITO_API_OPENROUTER_KEY: ${{ secrets.API_OPENROUTER_KEY || vars.API_OPENROUTER_KEY }} INFINITO_APP_DISCOVERY_RUNNER: host INFINITO_DOCKER_FILESYSTEM_ALLOWED: ${{ inputs.filesystem }} INFINITO_LIFECYCLES: ${{ inputs.lifecycles }} diff --git a/compose.yml b/compose.yml index f7d2be5738..635f1cea9a 100644 --- a/compose.yml +++ b/compose.yml @@ -71,6 +71,9 @@ services: - letsencrypt:/etc/letsencrypt/ environment: + INFINITO_API_ANTHROPIC_KEY: ${INFINITO_API_ANTHROPIC_KEY} + INFINITO_API_OPENAI_KEY: ${INFINITO_API_OPENAI_KEY} + INFINITO_API_OPENROUTER_KEY: ${INFINITO_API_OPENROUTER_KEY} INFINITO_BUILDKIT_ENABLED: ${INFINITO_BUILDKIT_ENABLED} INFINITO_CONTAINER_CA_ENABLED: ${INFINITO_CONTAINER_CA_ENABLED} INFINITO_DIR_VAR_LIB: ${INFINITO_DIR_VAR_LIB} diff --git a/default.env b/default.env index 6f4233db7a..440e46ac39 100644 --- a/default.env +++ b/default.env @@ -16,6 +16,15 @@ # Git repository holding the operator's terminal aliases (an `aliases` file at repo root); developers override this with their own repo. INFINITO_ALIAS_REPOSITORY=https://github.com/kevinveenbirkenbach/alias +# Anthropic API key the development inventory feeds into API.anthropic.api_key; empty leaves the anthropic routes unpublished. +INFINITO_API_ANTHROPIC_KEY="" + +# OpenAI API key the development inventory feeds into API.openai.api_key; empty leaves the openai routes unpublished. +INFINITO_API_OPENAI_KEY="" + +# OpenRouter API key the development inventory feeds into API.openrouter.api_key; empty leaves openrouter/auto unpublished. +INFINITO_API_OPENROUTER_KEY="" + # Where scripts/meta/resolve/apps.sh runs the meta-CLI: host | docker. INFINITO_APP_DISCOVERY_RUNNER=docker diff --git a/docs/contributing/design/role/services/layout.md b/docs/contributing/design/role/services/layout.md index 3ac1b7c16e..75c8677b68 100644 --- a/docs/contributing/design/role/services/layout.md +++ b/docs/contributing/design/role/services/layout.md @@ -87,6 +87,8 @@ The unified schema supports: - **Nested keys.** Both flat and nested credential keys are accepted, so e.g. `recaptcha.key` and `recaptcha.secret` remain nested. - **`algorithm:` defaults to `plain`** when the field is omitted. +- **`type:` (optional)** is one of `string`, `integer`, `boolean`. A supplied value of another type aborts inventory creation. +- **`regex:` (optional)** is a pattern the supplied value MUST match in full (`re.fullmatch`). Anchors are implicit; `sk-[A-Za-z0-9]{16,}` needs no `^` or `$`. - **`default:` (optional)** is a Jinja string used as the credential's value when the inventory does not provide one. - `default:` is **NOT rendered at inventory creation time.** The literal Jinja string is written verbatim into the inventory so that referenced variables (`CAPTCHA.RECAPTCHA.KEY`, `lookup(...)`, โ€ฆ) resolve only at deploy/runtime when those variables are actually defined. - `default:` values are **NOT validated.** `validation:` only applies to user-provided values, so the schema default is exempt. @@ -108,6 +110,58 @@ credentials: default: "{{ CAPTCHA.RECAPTCHA.SECRET | default('') }}" ``` +### What is checked, and when ๐Ÿ”Ž + +`type:` and `regex:` are enforced by `validate_supplied_value()` in +[inventory.py](../../../../../utils/manager/inventory.py) at inventory-creation +time, on every path where a value arrives from outside: + +| Source | Checked | +|---|---| +| `--set applications..secrets.credentials.=` | yes | +| A value already present in the inventory | yes | +| The literal written from `default:` | no | +| A value produced from `algorithm:` | no | + +A generated value is exempt because the algorithm that produced it is named by +the same schema, and `generate_value()` takes no schema argument. + +An empty string means "not configured" and passes every check, which is what +makes an optional provider key expressible: declare the shape, leave the value +empty, and the consumer gates on emptiness. + +`validation.min_length` is declared in many role schemas but enforced nowhere. +Do not read it as a guarantee. + +### Worked Example: an optional external key + +```yaml +# roles/svc-ai-litellm/meta/secrets.yml +credentials: + openai_api_key: + description: "OpenAI API key; empty leaves the openai/* models unpublished" + type: "string" + regex: "\\S{20,}" + default: "{{ API.openai.api_key }}" +``` + +The `default:` points at the central `API` block in +[group_vars/all/18_api.yml](../../../../../group_vars/all/18_api.yml), so an +operator fills the provider in one place while the role keeps the shape the +value must have. + +### Write a pattern for the paste, not for the vendor ๐Ÿšซ + +`regex:` MUST NOT pin a vendor's key prefix. OpenAI moved issuance from `sk-` to +`sk-proj-` without notice, and Anthropic issues `sk-ant-api03-`; a schema that +encoded either would reject a valid key and abort inventory creation, which is a +hard failure for a value that was correct. + +Write the pattern against the mistakes a paste makes โ€” a trailing newline, an +embedded space, a truncated copy, a `` โ€” and leave the provider's +format alone. A key pasted into the wrong provider's slot is not caught here; +it is caught at deploy time, where the route it publishes answers nothing. + Flat schema entries keep the same shape: ```yaml diff --git a/docs/contributing/tools/github/actions/configuration.md b/docs/contributing/tools/github/actions/configuration.md index e6d8cd29e8..2c852d94e2 100644 --- a/docs/contributing/tools/github/actions/configuration.md +++ b/docs/contributing/tools/github/actions/configuration.md @@ -12,6 +12,31 @@ Repository variables are set under **Settings โ†’ Secrets and variables โ†’ Acti | `CI_RUN_ON_MAIN` | [entry-push-latest.yml](../../../../../.github/workflows/entry-push-latest.yml) | Pushes to `main` skip CI | `true` to run CI on `main` pushes too | | `CI_ENABLE_AUTO_UPDATES` | [cron-update.yml](../../../../../.github/workflows/cron-update.yml), [entry-pr-open-dependabot-close.yml](../../../../../.github/workflows/entry-pr-open-dependabot-close.yml) | Update jobs skipped; Dependabot PRs auto-closed | `true` to allow update PRs (workflow-driven and Dependabot) | | `INFINITO_PLAYWRIGHT_KEEP` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | Playwright keeps trace, screenshot and video only when a test fails | `true` to keep them for every test (passing runs included) | +| `API_OPENAI_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | The gateway publishes no `openai/*` model, and the CLI test asserts their absence | An OpenAI key, to deploy and test the route | +| `API_ANTHROPIC_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | The gateway publishes no `anthropic/*` model | An Anthropic key | +| `API_OPENROUTER_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | The gateway publishes no `openrouter/auto` model | An OpenRouter key | + +## Provider keys: variable or secret? ๐Ÿ”‘ + +The three `API_*_KEY` entries are read as `${{ secrets.X || vars.X }}`, so a +secret of the same name wins and the variable is the fallback. Prefer the +secret: a repository **variable is not masked**, and the development inventory +sets `MASK_CREDENTIALS_IN_LOGS: false`, so a key supplied as a variable can be +printed by a task that echoes its own arguments. + +An unset key is not an error. The provider's route stays unpublished, the +gateway serves whatever local backend the round deploys, and +[the litellm CLI test](../../../../../roles/svc-ai-litellm/files/test/probe.py) +asserts the absence rather than skipping. + +They travel as `INFINITO_API_OPENAI_KEY`, `INFINITO_API_ANTHROPIC_KEY` and +`INFINITO_API_OPENROUTER_KEY` through the workflow `env:` block, the passthrough +allowlist in [passthrough.py](../../../../../utils/env/handlers/passthrough.py), +the `infinito` service environment in +[compose.yml](../../../../../compose.yml), and land in `API..api_key` +in the development inventory, whose path is the SPOT behind +`INFINITO_INVENTORY_VARS_FILE`. Skipping the compose forwarding makes them +resolve to empty inside the container without any error. ## Cancelling in-progress runs ๐Ÿ›‘ diff --git a/docs/contributing/tools/github/actions/secrets.md b/docs/contributing/tools/github/actions/secrets.md index 6679053894..4343c33419 100644 --- a/docs/contributing/tools/github/actions/secrets.md +++ b/docs/contributing/tools/github/actions/secrets.md @@ -10,6 +10,29 @@ Repository secrets MUST be set under **Settings โ†’ Secrets and variables โ†’ Ac |---|---|---|---| | `BOT_APP_CLIENT_ID` | [cron-update.yml](../../../../../.github/workflows/cron-update.yml) | GitHub App Client ID (OAuth-style `Ivโ€ฆ` identifier shown on the App's General page) used to mint a short-lived installation token for the update PR. | `CI_ENABLE_AUTO_UPDATES == 'true'` | | `BOT_APP_PRIVATE_KEY` | [cron-update.yml](../../../../../.github/workflows/cron-update.yml) | PEM-encoded private key of the same GitHub App. Used to sign the JWT that exchanges for the installation token. | `CI_ENABLE_AUTO_UPDATES == 'true'` | +| `API_OPENAI_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | Publishes the `openai/*` models on the LiteLLM gateway and lets the CLI test exercise the route. | Never. Unset leaves the route unpublished. | +| `API_ANTHROPIC_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | Publishes the `anthropic/*` models. | Never. | +| `API_OPENROUTER_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | Publishes `openrouter/auto`. | Never. | + +## `API_OPENAI_KEY`, `API_ANTHROPIC_KEY`, `API_OPENROUTER_KEY` ๐Ÿค– + +These are optional and fail soft: unset means the provider's route is never +published, and the deployment falls back to whatever local backend the round +deploys. The workflow reads each as `${{ secrets.X || vars.X }}`, so the same +name works as a repository variable; see +[configuration.md](configuration.md#provider-keys-variable-or-secret-) for why +the secret is the better half of that pair. + +Set them the same way as the App credentials: **Settings โ†’ Secrets and variables +โ†’ Actions โ†’ Secrets โ†’ New repository secret**, one per provider, the value being +the provider's key verbatim. + +The shape each key must have is declared per role rather than here, in +[roles/svc-ai-litellm/meta/secrets.yml](../../../../../roles/svc-ai-litellm/meta/secrets.yml) +with a `type:` and a `regex:`. The pattern rejects a truncated or +whitespace-carrying paste and deliberately accepts any vendor prefix, so a +provider changing its issuance format cannot turn a valid key into a failed +inventory run. ## `BOT_APP_CLIENT_ID` and `BOT_APP_PRIVATE_KEY` ๐Ÿค– diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index 7906138287..d68976e3c0 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -21,10 +21,14 @@ AI_LOCAL_MODELS: - alias: nomic-embed-text:latest name: nomic-embed-text-v1.5 +AI_REMOTE_ALIASES: >- + {{ ['openai/gpt-4o-mini'] * ((API.openai.api_key | string | length > 0) | int) + + ['anthropic/claude-sonnet-4-5'] * ((API.anthropic.api_key | string | length > 0) | int) + + ['openrouter/auto'] * ((API.openrouter.api_key | string | length > 0) | int) }} + LITELLM_CHAT_MODEL: "{{ (lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') | map(attribute='alias') | reject('search', 'embed') | list | first | default('')) if 'svc-ai-ollama' in LITELLM_BACKEND_ROLES else ((lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') | map(attribute='alias') - | reject('search', 'embed') | list | first | default('')) if 'svc-ai-lmstudio' in LITELLM_BACKEND_ROLES else ('openrouter/auto' if ((OPENROUTER_API_KEY | string) - | length > 0) else '')) }}" + | reject('search', 'embed') | list | first | default('')) if 'svc-ai-lmstudio' in LITELLM_BACKEND_ROLES else (AI_REMOTE_ALIASES | first | default(''))) }}" LITELLM_CHAT_MODEL_SERVED: >- {{ (('svc-ai-ollama' in LITELLM_BACKEND_ROLES) @@ -33,12 +37,10 @@ LITELLM_CHAT_MODEL_SERVED: >- or (('svc-ai-lmstudio' in LITELLM_BACKEND_ROLES) and ((lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') | map(attribute='alias') | reject('search', 'embed') | list) | length > 0)) - or ((OPENROUTER_API_KEY | string) | length > 0) }} + or ((AI_REMOTE_ALIASES | length) > 0) }} LITELLM_USABLE: "{{ ('svc-ai-litellm' in group_names) and (LITELLM_CHAT_MODEL_SERVED | bool) }}" -OPENROUTER_API_KEY: "" - MCP_DISCOVERY: "{{ lookup('mcp_servers') }}" MCP_DISCOVERED_SERVERS: "{{ MCP_DISCOVERY.selected }}" MCP_RECONCILE_STRICT: false diff --git a/group_vars/all/18_api.yml b/group_vars/all/18_api.yml index 7b05738278..28ecf9d1fb 100644 --- a/group_vars/all/18_api.yml +++ b/group_vars/all/18_api.yml @@ -6,8 +6,14 @@ # encrypted) inventory; access ONLY via lookup('api', '.'). # nocheck: unused-var: consumed by the api lookup plugin (plugins/lookup/api.py) via lookup('api', ...), not referenced directly in YAML/Jinja. API: + anthropic: + api_key: "" cloudflare: api_token: "" + openai: + api_key: "" + openrouter: + api_key: "" telegram: bot_token: "" api_id: "" diff --git a/inventories/development/default.yml b/inventories/development/default.yml index b7aa882ca4..eb5f74116e 100644 --- a/inventories/development/default.yml +++ b/inventories/development/default.yml @@ -39,6 +39,14 @@ AI_LOCAL_MODELS_CI: - alias: qwen2.5:0.5b name: qwen2.5-0.5b-instruct +API: + openai: + api_key: "{{ lookup('env', 'INFINITO_API_OPENAI_KEY') }}" + anthropic: + api_key: "{{ lookup('env', 'INFINITO_API_ANTHROPIC_KEY') }}" + openrouter: + api_key: "{{ lookup('env', 'INFINITO_API_OPENROUTER_KEY') }}" + OIDC_OVERRIDE: CLIENT: SECRET: VERYUNSAFESAMPLEPASSWORD1234 diff --git a/roles/svc-ai-litellm/README.md b/roles/svc-ai-litellm/README.md index a87bcb9b7b..a975a2d0b9 100644 --- a/roles/svc-ai-litellm/README.md +++ b/roles/svc-ai-litellm/README.md @@ -6,7 +6,7 @@ ## Overview -This role deploys LiteLLM as a shared gateway container in Compose and Swarm deployments, backed by the central PostgreSQL service. The gateway is headless: it binds its HTTP port on the container host and claims no domain of its own, and the browser-facing surface is published separately by the LiteLLM Admin UI. It renders the model list from the interchangeable local backends available on the host, Ollama and LM Studio, and adds an OpenRouter entry when an OpenRouter API key is configured. Once the gateway is up, it provisions one virtual key per consuming application through the gateway admin API. +This role deploys LiteLLM as a shared gateway container in Compose and Swarm deployments, backed by the central PostgreSQL service. The gateway is headless: it binds its HTTP port on the container host and claims no domain of its own, and the browser-facing surface is published separately by the LiteLLM Admin UI. It renders the model list from the interchangeable local backends available on the host, Ollama and LM Studio, and adds the OpenAI, Anthropic and OpenRouter entries whose API key is configured. Once the gateway is up, it provisions one virtual key per consuming application through the gateway admin API. ## Cosmos @@ -63,7 +63,8 @@ Solid `1:1` edges are fixed relationships; dashed `0..1` edges are conditional ( ## Features - **OpenAI-compatible API:** One HTTP endpoint serves every model listed in the gateway configuration. -- **Backend routing:** Model entries are generated for Ollama and LM Studio when those services run on the host, and for OpenRouter when its API key is set. +- **Backend routing:** Model entries are generated for Ollama and LM Studio when those services run on the host, under the shared alias each model carries so a consumer names one model whichever local backend answers it. +- **Remote providers:** OpenAI, Anthropic and OpenRouter each add their models when a key is configured. The keys are declared in `meta/secrets.yml` with the `type` and `regex` their value must satisfy, and default to the central `API..api_key` entry; an unset key leaves that provider's routes unpublished. - **Per-consumer virtual keys:** Each consuming application receives its own virtual key, created through the gateway admin API under an alias naming that application. - **File-based model list:** The model list is mounted as a read-only config file, with database-stored model entries turned off. - **Managed credentials:** The gateway master key and the admin UI password are generated and kept as role credentials, and the admin UI username is the platform administrator name. diff --git a/roles/svc-ai-litellm/files/test/probe.py b/roles/svc-ai-litellm/files/test/probe.py index 551d61bfbb..f6ad07ce32 100644 --- a/roles/svc-ai-litellm/files/test/probe.py +++ b/roles/svc-ai-litellm/files/test/probe.py @@ -13,6 +13,8 @@ Env (rendered into test.env from templates/test.env.j2): LMSTUDIO_ALIASES JSON list of aliases only LM Studio provides, rendered whether or not the backend is deployed so the absent case stays assertable + REMOTE_ALIASES JSON list of aliases a configured provider key publishes + (OpenAI, Anthropic, OpenRouter) OLLAMA_ENABLED true|false LMSTUDIO_ENABLED true|false RETRIES completion attempts (default 10) @@ -28,8 +30,6 @@ import time import urllib.error import urllib.request -OPENROUTER_MODEL = "openrouter/auto" - def evaluate( *, @@ -40,6 +40,7 @@ def evaluate( ollama_enabled: bool, lmstudio_enabled: bool, lmstudio_aliases: list[str], + remote_aliases: list[str], ) -> list[str]: """Every routing violation the published model list shows, as messages. @@ -51,6 +52,7 @@ def evaluate( ollama_enabled: svc-ai-ollama is deployed on the gateway's host. lmstudio_enabled: svc-ai-lmstudio is deployed on the gateway's host. lmstudio_aliases: aliases only LM Studio provides, whatever is deployed. + remote_aliases: aliases a configured remote provider key publishes. Returns: One message per violation; empty when the routing is consistent. @@ -76,7 +78,7 @@ def evaluate( f"{sorted(exclusive & served)}; a route stands without its backend" ) - unbacked = sorted(served - exclusive - {OPENROUTER_MODEL}) + unbacked = sorted(served - exclusive - set(remote_aliases)) if not ollama_enabled and unbacked: failures.append( f"svc-ai-ollama is not deployed yet the gateway publishes {unbacked}" @@ -164,6 +166,7 @@ def main() -> int: ollama_enabled=os.environ["OLLAMA_ENABLED"] == "true", lmstudio_enabled=os.environ["LMSTUDIO_ENABLED"] == "true", lmstudio_aliases=json.loads(os.environ["LMSTUDIO_ALIASES"]), + remote_aliases=json.loads(os.environ["REMOTE_ALIASES"]), ) if chat_model_served and not failures: diff --git a/roles/svc-ai-litellm/files/test/test.sh b/roles/svc-ai-litellm/files/test/test.sh index 1f733cb4e9..f8e8a895f1 100644 --- a/roles/svc-ai-litellm/files/test/test.sh +++ b/roles/svc-ai-litellm/files/test/test.sh @@ -13,6 +13,7 @@ # LITELLM_CHAT_MODEL_SERVED true|false # LITELLM_EXPECTED_MODELS JSON list the config template published # LITELLM_LMSTUDIO_ALIASES JSON list of aliases only LM Studio provides +# LITELLM_REMOTE_ALIASES JSON list of aliases a configured provider key adds # LITELLM_OLLAMA_ENABLED true|false # LITELLM_LMSTUDIO_ENABLED true|false # READY_RETRIES gateway readiness attempts (default 30) @@ -54,6 +55,7 @@ probe() { -e "CHAT_MODEL_SERVED=${LITELLM_CHAT_MODEL_SERVED}" \ -e "EXPECTED_MODELS=${LITELLM_EXPECTED_MODELS}" \ -e "LMSTUDIO_ALIASES=${LITELLM_LMSTUDIO_ALIASES}" \ + -e "REMOTE_ALIASES=${LITELLM_REMOTE_ALIASES}" \ -e "OLLAMA_ENABLED=${LITELLM_OLLAMA_ENABLED}" \ -e "LMSTUDIO_ENABLED=${LITELLM_LMSTUDIO_ENABLED}" \ "${LITELLM_CONTAINER}" "$(interpreter)" - <"${here}/probe.py" diff --git a/roles/svc-ai-litellm/meta/secrets.yml b/roles/svc-ai-litellm/meta/secrets.yml index 3a0c374abc..47c983f31c 100644 --- a/roles/svc-ai-litellm/meta/secrets.yml +++ b/roles/svc-ai-litellm/meta/secrets.yml @@ -10,3 +10,19 @@ credentials: algorithm: "alphanumeric" validation: min_length: 32 + openai_api_key: + description: "OpenAI API key; empty leaves the openai/* models unpublished. The pattern rejects a truncated or whitespace-carrying paste, not a vendor prefix: + OpenAI moved from sk- to sk-proj- without notice, and a schema that pinned the prefix would reject a valid key" + type: "string" + regex: "\\S{20,}" + default: "{{ API.openai.api_key }}" + anthropic_api_key: + description: "Anthropic API key; empty leaves the anthropic/* models unpublished" + type: "string" + regex: "\\S{20,}" + default: "{{ API.anthropic.api_key }}" + openrouter_api_key: + description: "OpenRouter API key; empty leaves openrouter/auto unpublished" + type: "string" + regex: "\\S{20,}" + default: "{{ API.openrouter.api_key }}" diff --git a/roles/svc-ai-litellm/templates/config.yaml.j2 b/roles/svc-ai-litellm/templates/config.yaml.j2 index 07faa6deec..6d7d02a8be 100644 --- a/roles/svc-ai-litellm/templates/config.yaml.j2 +++ b/roles/svc-ai-litellm/templates/config.yaml.j2 @@ -1,6 +1,7 @@ {% set _ollama_models = lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') if 'svc-ai-ollama' in group_names else [] %} {% set _lmstudio_models = lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') if 'svc-ai-lmstudio' in group_names else [] %} -{% set has_backend = _ollama_models or _lmstudio_models or LITELLM_OPENROUTER_API_KEY %} +{% set _remote_models = LITELLM_REMOTE_BACKENDS | selectattr('key') | list %} +{% set has_backend = _ollama_models or _lmstudio_models or _remote_models %} model_list:{{ ' []' if not has_backend else '' }} {% for _model in _ollama_models %} - model_name: "{{ _model.alias }}" @@ -21,12 +22,14 @@ model_list:{{ ' []' if not has_backend else '' }} max_tokens: {{ LITELLM_MAX_OUTPUT_TOKENS }} timeout: {{ LITELLM_UPSTREAM_TIMEOUT }} {% endfor %} -{% if LITELLM_OPENROUTER_API_KEY %} - - model_name: openrouter/auto +{% for _backend in _remote_models %} + - model_name: "{{ _backend.alias }}" litellm_params: - model: openrouter/openrouter/auto - api_key: os.environ/OPENROUTER_API_KEY -{% endif %} + model: "{{ _backend.model }}" + api_key: os.environ/{{ _backend.env }} + max_tokens: {{ LITELLM_MAX_OUTPUT_TOKENS }} + timeout: {{ LITELLM_UPSTREAM_TIMEOUT }} +{% endfor %} litellm_settings: drop_params: true diff --git a/roles/svc-ai-litellm/templates/env.j2 b/roles/svc-ai-litellm/templates/env.j2 index c435fd3aa7..e77c43da8e 100644 --- a/roles/svc-ai-litellm/templates/env.j2 +++ b/roles/svc-ai-litellm/templates/env.j2 @@ -5,9 +5,9 @@ DATABASE_URL={{ lookup('database', application_id, 'url_full') | dotenv_quote }} STORE_MODEL_IN_DB=False UI_USERNAME={{ lookup('users', 'administrator').username }} UI_PASSWORD={{ lookup('config', application_id, 'secrets.credentials.ui_password') | dotenv_quote }} -{% if LITELLM_OPENROUTER_API_KEY %} -OPENROUTER_API_KEY={{ LITELLM_OPENROUTER_API_KEY | dotenv_quote }} -{% endif %} +{% for _backend in LITELLM_REMOTE_BACKENDS if _backend.key %} +{{ _backend.env }}={{ _backend.key | dotenv_quote }} +{% endfor %} {% if LITELLM_UI_SSO | bool %} PROXY_BASE_URL={{ LITELLM_UI_BASE_URL }} PROXY_ADMIN_ID={{ LITELLM_UI_ADMIN }} diff --git a/roles/svc-ai-litellm/templates/test.env.j2 b/roles/svc-ai-litellm/templates/test.env.j2 index fe917bb9db..47d7508fe7 100644 --- a/roles/svc-ai-litellm/templates/test.env.j2 +++ b/roles/svc-ai-litellm/templates/test.env.j2 @@ -1,9 +1,8 @@ {%- set _ollama_models = (lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') | map(attribute='alias') | list) if 'svc-ai-ollama' in group_names else [] -%} {%- set _lmstudio_declared = lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') | map(attribute='alias') | list -%} {%- set _lmstudio_only = _lmstudio_declared | reject('in', _ollama_models) | list -%} -{%- set _expected = _ollama_models - + (_lmstudio_only if 'svc-ai-lmstudio' in group_names else []) - + (['openrouter/auto'] if LITELLM_OPENROUTER_API_KEY else []) -%} +{%- set _remote = LITELLM_REMOTE_BACKENDS | selectattr('key') | map(attribute='alias') | list -%} +{%- set _expected = _ollama_models + (_lmstudio_only if 'svc-ai-lmstudio' in group_names else []) + _remote -%} LITELLM_CONTAINER={{ CLI_LOCAL_CID }} LITELLM_PORT={{ LITELLM_PORT }} LITELLM_MASTER_KEY_B64={{ LITELLM_MASTER_KEY | b64encode }} @@ -12,4 +11,5 @@ LITELLM_CHAT_MODEL_SERVED={{ LITELLM_CHAT_MODEL_SERVED | bool | string | lower } LITELLM_OLLAMA_ENABLED={{ ('svc-ai-ollama' in group_names) | string | lower }} LITELLM_LMSTUDIO_ENABLED={{ ('svc-ai-lmstudio' in group_names) | string | lower }} LITELLM_LMSTUDIO_ALIASES='{{ _lmstudio_only | to_json }}' +LITELLM_REMOTE_ALIASES='{{ _remote | to_json }}' LITELLM_EXPECTED_MODELS='{{ _expected | to_json }}' diff --git a/roles/svc-ai-litellm/vars/main.yml b/roles/svc-ai-litellm/vars/main.yml index 8d9da03e66..a2cbbf9cef 100644 --- a/roles/svc-ai-litellm/vars/main.yml +++ b/roles/svc-ai-litellm/vars/main.yml @@ -9,7 +9,22 @@ LITELLM_MAX_OUTPUT_TOKENS: "{{ lookup('config', application_id, 'services.litell LITELLM_UPSTREAM_TIMEOUT: "{{ lookup('config', application_id, 'services.litellm.upstream_timeout') }}" LITELLM_CONFIG_PATH_HOST: "{{ [ lookup('container', application_id, 'directories.config'), 'config.yaml' ] | path_join }}" LITELLM_CONFIG_PATH_DOCKER: "{{ lookup('volume', application_id, 'litellm_config').target }}" -LITELLM_OPENROUTER_API_KEY: "{{ OPENROUTER_API_KEY }}" +LITELLM_OPENAI_API_KEY: "{{ lookup('config', application_id, 'secrets.credentials.openai_api_key') }}" +LITELLM_ANTHROPIC_API_KEY: "{{ lookup('config', application_id, 'secrets.credentials.anthropic_api_key') }}" +LITELLM_OPENROUTER_API_KEY: "{{ lookup('config', application_id, 'secrets.credentials.openrouter_api_key') }}" +LITELLM_REMOTE_BACKENDS: + - alias: openai/gpt-4o-mini + model: openai/gpt-4o-mini + key: "{{ LITELLM_OPENAI_API_KEY }}" + env: OPENAI_API_KEY + - alias: anthropic/claude-sonnet-4-5 + model: anthropic/claude-sonnet-4-5 + key: "{{ LITELLM_ANTHROPIC_API_KEY }}" + env: ANTHROPIC_API_KEY + - alias: openrouter/auto + model: openrouter/openrouter/auto + key: "{{ LITELLM_OPENROUTER_API_KEY }}" + env: OPENROUTER_API_KEY LITELLM_UI_ROLE: "web-app-litellm" LITELLM_UI_SSO: "{{ (LITELLM_UI_ROLE in group_names) and (lookup('sso', LITELLM_UI_ROLE, 'is_oidc_native') | bool) }}" LITELLM_UI_BASE_URL: "{{ lookup('tls', LITELLM_UI_ROLE, 'url.base') }}" diff --git a/roles/web-app-mattermost/tasks/utils/ai_gateway.yml b/roles/web-app-mattermost/tasks/utils/ai_gateway.yml index be9ad2edfb..12cf167c38 100644 --- a/roles/web-app-mattermost/tasks/utils/ai_gateway.yml +++ b/roles/web-app-mattermost/tasks/utils/ai_gateway.yml @@ -107,7 +107,7 @@ model {{ LITELLM_CHAT_MODEL }}), so prompts would leave the deployment, use a model the gateway does not serve, or carry a stale virtual key. An empty model means the gateway serves none: neither svc-ai-ollama nor - svc-ai-lmstudio is in this deployment and OPENROUTER_API_KEY is empty. + svc-ai-lmstudio is in this deployment and no remote provider key is set. - name: "๐Ÿ”Ž Mattermost | Wait for the Agents plugin to provision its bot account" database_query: diff --git a/tests/unit/python/group_vars/test_litellm_chat_model.py b/tests/unit/python/group_vars/test_litellm_chat_model.py index 4b4022ceb1..a348fb6268 100644 --- a/tests/unit/python/group_vars/test_litellm_chat_model.py +++ b/tests/unit/python/group_vars/test_litellm_chat_model.py @@ -45,7 +45,7 @@ class TestLitellmChatModel(unittest.TestCase): self.env.from_string(self.source[name]) .render( LITELLM_BACKEND_ROLES=list(roles), - OPENROUTER_API_KEY=api_key, + AI_REMOTE_ALIASES=(["openrouter/auto"] if api_key else []), lookup=_stub_lookup( [{"alias": alias, "name": alias} for alias in preload_models], [ diff --git a/tests/unit/python/roles/svc-ai-litellm/files/test_probe_routing.py b/tests/unit/python/roles/svc-ai-litellm/files/test_probe_routing.py index f16148508b..b2073361b1 100644 --- a/tests/unit/python/roles/svc-ai-litellm/files/test_probe_routing.py +++ b/tests/unit/python/roles/svc-ai-litellm/files/test_probe_routing.py @@ -26,6 +26,7 @@ def verdict(**overrides) -> list[str]: "ollama_enabled": False, "lmstudio_enabled": False, "lmstudio_aliases": [LMSTUDIO_ONLY], + "remote_aliases": [], } kwargs.update(overrides) return probe.evaluate(**kwargs) @@ -155,12 +156,54 @@ class TestBothBackendsDeployment(unittest.TestCase): ) +class TestRemoteProviders(unittest.TestCase): + """A configured provider key publishes its alias and serves the gateway.""" + + def test_a_keyed_provider_is_the_only_backend_needed(self) -> None: + self.assertEqual( + verdict( + served={"openai/gpt-4o-mini"}, + expected=["openai/gpt-4o-mini"], + chat_model="openai/gpt-4o-mini", + chat_model_served=True, + remote_aliases=["openai/gpt-4o-mini"], + ), + [], + ) + + def test_a_remote_alias_is_not_mistaken_for_an_ollama_model(self) -> None: + self.assertEqual( + verdict( + served={"anthropic/claude-sonnet-4-5", ALIAS}, + expected=["anthropic/claude-sonnet-4-5", ALIAS], + chat_model=ALIAS, + chat_model_served=True, + lmstudio_enabled=True, + lmstudio_aliases=[ALIAS], + remote_aliases=["anthropic/claude-sonnet-4-5"], + ), + [], + ) + + def test_a_declared_provider_the_gateway_dropped_fails(self) -> None: + failures = verdict( + served=set(), + expected=["openrouter/auto"], + chat_model="openrouter/auto", + chat_model_served=True, + remote_aliases=["openrouter/auto"], + ) + self.assertTrue( + any("does not serve" in failure for failure in failures), failures + ) + + class TestNoBackendDeployment(unittest.TestCase): def test_an_empty_gateway_passes(self) -> None: self.assertEqual(verdict(), []) def test_any_published_model_fails(self) -> None: - failures = verdict(served={probe.OPENROUTER_MODEL}) + failures = verdict(served={"openrouter/auto"}) self.assertTrue( any("no backend is deployed" in failure for failure in failures), failures ) diff --git a/tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py b/tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py index 2c97d52f4c..ef5fd8c093 100644 --- a/tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py +++ b/tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py @@ -39,13 +39,32 @@ def _stub_lookup(ollama_models, lmstudio_models): return lookup -def render(*, ollama=(), lmstudio=(), api_key=""): +REMOTE_BACKENDS = ( + ("openai/gpt-4o-mini", "openai/gpt-4o-mini", "OPENAI_API_KEY"), + ("anthropic/claude-sonnet-4-5", "anthropic/claude-sonnet-4-5", "ANTHROPIC_API_KEY"), + ("openrouter/auto", "openrouter/openrouter/auto", "OPENROUTER_API_KEY"), +) + + +def _remote(keys): + """The LITELLM_REMOTE_BACKENDS list vars/main.yml builds, with these keys. + + Args: + keys: env-name -> key value; a provider absent from it stays unkeyed. + """ + return [ + {"alias": alias, "model": model, "env": env, "key": keys.get(env, "")} + for alias, model, env in REMOTE_BACKENDS + ] + + +def render(*, ollama=(), lmstudio=(), keys=None): """The rendered config as a parsed mapping. Args: ollama: preload entries svc-ai-ollama declares; empty means not deployed. lmstudio: preload entries svc-ai-lmstudio declares; empty means not deployed. - api_key: the OpenRouter key, empty when none is configured. + keys: env-name -> provider key for the remote backends. """ group_names = ["svc-ai-litellm"] if ollama: @@ -60,7 +79,7 @@ def render(*, ollama=(), lmstudio=(), api_key=""): LMSTUDIO_BASE_LOCAL_URL=LMSTUDIO_URL, LITELLM_MAX_OUTPUT_TOKENS=512, LITELLM_UPSTREAM_TIMEOUT=60, - LITELLM_OPENROUTER_API_KEY=api_key, + LITELLM_REMOTE_BACKENDS=_remote(keys or {}), ) return yaml.safe_load( rendered @@ -119,8 +138,28 @@ class TestNoBackend(unittest.TestCase): def test_the_model_list_is_empty(self) -> None: self.assertEqual(render()["model_list"], []) - def test_the_openrouter_key_alone_publishes_its_route(self) -> None: - self.assertEqual(set(routes(render(api_key="sk-test"))), {"openrouter/auto"}) + def test_one_provider_key_alone_publishes_its_route(self) -> None: + published = routes(render(keys={"OPENROUTER_API_KEY": "sk-or-test"})) + self.assertEqual(set(published), {"openrouter/auto"}) + self.assertEqual( + published["openrouter/auto"]["api_key"], "os.environ/OPENROUTER_API_KEY" + ) + + def test_each_keyed_provider_gets_its_own_route(self) -> None: + published = routes( + render( + keys={ + "OPENAI_API_KEY": "sk-test", + "ANTHROPIC_API_KEY": "sk-ant-test", + } + ) + ) + self.assertEqual( + set(published), {"openai/gpt-4o-mini", "anthropic/claude-sonnet-4-5"} + ) + + def test_an_unkeyed_provider_publishes_nothing(self) -> None: + self.assertEqual(render(keys={"OPENAI_API_KEY": ""})["model_list"], []) if __name__ == "__main__": diff --git a/tests/unit/python/utils/manager/test_inventory_manager.py b/tests/unit/python/utils/manager/test_inventory_manager.py index 5740a345a5..19028fa34c 100644 --- a/tests/unit/python/utils/manager/test_inventory_manager.py +++ b/tests/unit/python/utils/manager/test_inventory_manager.py @@ -1,13 +1,16 @@ import base64 import tempfile from pathlib import Path +from typing import ClassVar from unittest import TestCase, main, mock from cryptography.hazmat.primitives.asymmetric import ec from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat +from utils.cache.files import PROJECT_ROOT +from utils.cache.yaml import load_yaml_any from utils.handler.vault import VaultScalar -from utils.manager.inventory import InventoryManager +from utils.manager.inventory import InventoryManager, validate_supplied_value from utils.manager.value_generator import ValueGenerator from utils.roles.mapping import ( ROLE_FILE_META_SECRETS, @@ -760,5 +763,84 @@ class TestInventoryManagerVariant(TestCase): self.assertEqual(cfg, {}) +class TestSuppliedValueValidation(TestCase): + """A schema that declares a shape must reject a value contradicting it. + + Without this the declaration is decoration: the 192 `validation:` blocks + already in the role schemas are enforced nowhere, and a `regex` would have + joined them. + """ + + SCHEMA: ClassVar[dict[str, str]] = { + "type": "string", + "regex": "sk-[A-Za-z0-9]{4,}", + } + + def test_a_matching_value_passes(self): + validate_supplied_value("credentials.api_key", "sk-abcd1234", self.SCHEMA) + + def test_an_empty_value_is_not_configured_and_passes(self): + validate_supplied_value("credentials.api_key", "", self.SCHEMA) + validate_supplied_value("credentials.api_key", None, self.SCHEMA) + + def test_a_value_failing_the_pattern_aborts(self): + with self.assertRaises(SystemExit): + validate_supplied_value("credentials.api_key", "nope", self.SCHEMA) + + def test_a_partial_match_is_not_enough(self): + with self.assertRaises(SystemExit): + validate_supplied_value("credentials.api_key", "sk-ab!!", self.SCHEMA) + + def test_a_value_of_the_wrong_type_aborts(self): + with self.assertRaises(SystemExit): + validate_supplied_value("credentials.port", 8080, {"type": "string"}) + + def test_an_unknown_declared_type_aborts(self): + with self.assertRaises(SystemExit): + validate_supplied_value("credentials.x", "v", {"type": "uuid"}) + + def test_a_schema_declaring_neither_accepts_anything(self): + validate_supplied_value("credentials.x", "anything", {"description": "free"}) + + +class TestProviderKeyPattern(TestCase): + """The shipped provider-key pattern must outlive a vendor's format change. + + OpenAI moved from `sk-` to `sk-proj-` without notice. A schema that pinned + the prefix would reject a valid key and abort inventory creation, so the + pattern checks what a paste can get wrong instead. + """ + + ACCEPTED: ClassVar[tuple[str, ...]] = ( + "sk-proj-AbCdEfGh1234567890xyz", + "sk-ant-api03-AbCdEfGh1234567890", + "sk-or-v1-AbCdEfGh1234567890abc", + "sk-AbCdEfGh1234567890legacyKey", + "some-future-format-2099-abcdef", + ) + REJECTED: ClassVar[tuple[str, ...]] = ( + "sk-proj-AbCdEfGh1234567890xyz\n", + "sk-short", + "", + "sk-proj-Ab Cd 1234567890 xyz", + ) + + def _schema(self) -> dict: + path = PROJECT_ROOT / "roles/svc-ai-litellm" / ROLE_FILE_META_SECRETS + return load_yaml_any(str(path))["credentials"]["openai_api_key"] + + def test_every_known_provider_form_is_accepted(self): + schema = self._schema() + for value in self.ACCEPTED: + with self.subTest(value=value): + validate_supplied_value("credentials.openai_api_key", value, schema) + + def test_a_broken_paste_is_rejected(self): + schema = self._schema() + for value in self.REJECTED: + with self.subTest(value=value), self.assertRaises(SystemExit): + validate_supplied_value("credentials.openai_api_key", value, schema) + + if __name__ == "__main__": main() diff --git a/utils/env/handlers/passthrough.py b/utils/env/handlers/passthrough.py index b0a30ff3d0..f054fda41c 100644 --- a/utils/env/handlers/passthrough.py +++ b/utils/env/handlers/passthrough.py @@ -71,6 +71,9 @@ STATIC_KEYS = ( "INFINITO_ALIAS_REPOSITORY", "INFINITO_APT_UBUNTU_MIRRORS", "INFINITO_FORK_REPOSITORY_NAME", + "INFINITO_API_ANTHROPIC_KEY", + "INFINITO_API_OPENAI_KEY", + "INFINITO_API_OPENROUTER_KEY", "INFINITO_PLAYWRIGHT_KEEP", "INFINITO_PUBLISH_PORTS", "INFINITO_SKILLS_REPOSITORY", diff --git a/utils/manager/inventory.py b/utils/manager/inventory.py index ac3463d36d..94d0907cbf 100644 --- a/utils/manager/inventory.py +++ b/utils/manager/inventory.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import sys from typing import TYPE_CHECKING, Any @@ -19,7 +20,20 @@ from utils.roles.mapping import ROLE_FILE_META_SECRETS, ROLE_FILE_VARS_MAIN if TYPE_CHECKING: from pathlib import Path -_CREDENTIAL_LEAF_MARKERS = ("description", "algorithm", "validation", "default") +_CREDENTIAL_LEAF_MARKERS = ( + "description", + "algorithm", + "validation", + "default", + "type", + "regex", +) + +_SCHEMA_TYPES: dict[str, type | tuple[type, ...]] = { + "string": str, + "integer": int, + "boolean": bool, +} def _is_credential_leaf(node: Any) -> bool: @@ -28,6 +42,51 @@ def _is_credential_leaf(node: Any) -> bool: ) +def validate_supplied_value(full_key: str, value: Any, meta: dict[str, Any]) -> None: + """Hold an operator-supplied credential to the schema it declares. + + Only supplied values are checked: a generated one comes from the algorithm + the same schema names, and the generators take no schema argument. + + Args: + full_key: dotted schema path, used in the failure message. + value: the value the operator supplied, empty meaning "not configured". + meta: the schema leaf, read for `type` and `regex`. + + Raises: + SystemExit: the value contradicts the declared type or pattern. + """ + if value in ("", None): + return + + declared = meta.get("type") + if declared: + expected = _SCHEMA_TYPES.get(str(declared)) + if expected is None: + print( + f"ERROR: '{full_key}' declares unknown type '{declared}'; " + f"known types are {', '.join(sorted(_SCHEMA_TYPES))}", + file=sys.stderr, + ) + sys.exit(1) + if not isinstance(value, expected): + print( + f"ERROR: '{full_key}' is declared {declared} but the supplied " + f"value is {type(value).__name__}", + file=sys.stderr, + ) + sys.exit(1) + + pattern = meta.get("regex") + if pattern and not re.fullmatch(str(pattern), str(value)): + print( + f"ERROR: the value supplied for '{full_key}' does not match the " + f"pattern its schema declares: {pattern}", + file=sys.stderr, + ) + sys.exit(1) + + def _meta_role_config(role_path: Path) -> dict[str, Any]: """Assemble the view of a role's config from its meta files. @@ -251,6 +310,9 @@ class InventoryManager: * `default:` (Jinja literal) is written verbatim and short-circuits algorithm-based generation. `validation:` is ignored for default-bearing entries. + * `type:` and `regex:` hold an operator-supplied value to the + shape the schema declares; a generated one is exempt because + the algorithm that produced it is named by the same schema. * Existing inventory values are preserved (no double-encryption, no overwrite of operator-supplied secrets). """ @@ -295,6 +357,7 @@ class InventoryManager: if "default" in meta: if isinstance(existing_value, str) and existing_value != "": + validate_supplied_value(full_key, existing_value, meta) return dest[key] = meta["default"] return @@ -306,7 +369,9 @@ class InventoryManager: if algorithm == "plain": if set_key in self.overrides: plain = self.overrides[set_key] + validate_supplied_value(full_key, plain, meta) elif isinstance(existing_value, str) and existing_value != "": + validate_supplied_value(full_key, existing_value, meta) return elif self.allow_empty_plain: plain = "" From d8c6bd242d1770923766aff8fb1dc73d9cefa3c0 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 19:59:10 +0200 Subject: [PATCH 017/197] test(ci): require every secret and variable a workflow reads to be documented An unset secret or variable resolves to the empty string, not to an error, so an undocumented one is invisible twice: the administrator cannot know to set it, and the run that needed it degrades quietly instead of failing. The guard reads every ${{ secrets.X }} and ${{ vars.X }} out of .github/workflows and demands the name in the page that owns it, secrets.md or configuration.md. It compares the reference against the page rather than the page against itself, so it also catches a doc naming something the workflows no longer read. Three gaps were already there. DOCKERHUB_USERNAME and DOCKERHUB_TOKEN authenticate the image mirror against Docker Hub and lift the anonymous pull-rate limit; both are declared required: false and the login step is gated on them being non-empty, so their absence does not fail a run, it walks the mirror into the rate limit. Neither was documented anywhere. PLAYWRIGHT_KEEP_ALL is the sharper one: configuration.md listed INFINITO_PLAYWRIGHT_KEEP, which is the name the workflow forwards it under, not the name an administrator sets. Following the documentation produced a variable nothing reads. Commented-out lines are skipped. cron-security-scorecard.yml carries `# repo_token: ${{ secrets.SCORECARD_TOKEN }}` as an option nobody has taken, and demanding a doc row for it would describe an input no run asks for. GITHUB_TOKEN is exempt in _PROVIDED_SECRETS because GitHub mints it per run; it is not administered and cannot be set. A second case asserts the scan finds workflow files at all, so the guard cannot pass vacuously. Co-Authored-By: Claude Opus 5 (1M context) --- .../tools/github/actions/configuration.md | 2 +- .../tools/github/actions/secrets.md | 2 + .../workflows/test_ci_inputs_documented.py | 96 +++++++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 tests/integration/meta/workflows/test_ci_inputs_documented.py diff --git a/docs/contributing/tools/github/actions/configuration.md b/docs/contributing/tools/github/actions/configuration.md index 2c852d94e2..f069964ae8 100644 --- a/docs/contributing/tools/github/actions/configuration.md +++ b/docs/contributing/tools/github/actions/configuration.md @@ -11,7 +11,7 @@ Repository variables are set under **Settings โ†’ Secrets and variables โ†’ Acti | `CI_SYNC_MAIN_SOURCE_REPOSITORY` | [entry-push-latest.yml](../../../../../.github/workflows/entry-push-latest.yml) | Syncs `main` from `infinito-nexus/core` before CI scope discovery | `/` to use another source, or `false`, empty, or the current repository to skip | | `CI_RUN_ON_MAIN` | [entry-push-latest.yml](../../../../../.github/workflows/entry-push-latest.yml) | Pushes to `main` skip CI | `true` to run CI on `main` pushes too | | `CI_ENABLE_AUTO_UPDATES` | [cron-update.yml](../../../../../.github/workflows/cron-update.yml), [entry-pr-open-dependabot-close.yml](../../../../../.github/workflows/entry-pr-open-dependabot-close.yml) | Update jobs skipped; Dependabot PRs auto-closed | `true` to allow update PRs (workflow-driven and Dependabot) | -| `INFINITO_PLAYWRIGHT_KEEP` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | Playwright keeps trace, screenshot and video only when a test fails | `true` to keep them for every test (passing runs included) | +| `PLAYWRIGHT_KEEP_ALL` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | Playwright keeps trace, screenshot and video only when a test fails | `true` to keep them for every test (passing runs included); the workflow forwards it as `INFINITO_PLAYWRIGHT_KEEP` | | `API_OPENAI_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | The gateway publishes no `openai/*` model, and the CLI test asserts their absence | An OpenAI key, to deploy and test the route | | `API_ANTHROPIC_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | The gateway publishes no `anthropic/*` model | An Anthropic key | | `API_OPENROUTER_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | The gateway publishes no `openrouter/auto` model | An OpenRouter key | diff --git a/docs/contributing/tools/github/actions/secrets.md b/docs/contributing/tools/github/actions/secrets.md index 4343c33419..54cefb8974 100644 --- a/docs/contributing/tools/github/actions/secrets.md +++ b/docs/contributing/tools/github/actions/secrets.md @@ -13,6 +13,8 @@ Repository secrets MUST be set under **Settings โ†’ Secrets and variables โ†’ Ac | `API_OPENAI_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | Publishes the `openai/*` models on the LiteLLM gateway and lets the CLI test exercise the route. | Never. Unset leaves the route unpublished. | | `API_ANTHROPIC_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | Publishes the `anthropic/*` models. | Never. | | `API_OPENROUTER_KEY` | [call-test-deploy.yml](../../../../../.github/workflows/call-test-deploy.yml) | Publishes `openrouter/auto`. | Never. | +| `DOCKERHUB_USERNAME` | [cron-images-mirror-all.yml](../../../../../.github/workflows/cron-images-mirror-all.yml), [call-images-mirror-missing.yml](../../../../../.github/workflows/call-images-mirror-missing.yml) | Authenticates the mirror job against Docker Hub, which raises the anonymous pull-rate limit the mirror would otherwise hit. | Never. Both workflows declare the secret `required: false` and gate the login step on both values being non-empty. | +| `DOCKERHUB_TOKEN` | [cron-images-mirror-all.yml](../../../../../.github/workflows/cron-images-mirror-all.yml), [call-images-mirror-missing.yml](../../../../../.github/workflows/call-images-mirror-missing.yml) | The access token paired with `DOCKERHUB_USERNAME`. | Never, and only together with the username. | ## `API_OPENAI_KEY`, `API_ANTHROPIC_KEY`, `API_OPENROUTER_KEY` ๐Ÿค– diff --git a/tests/integration/meta/workflows/test_ci_inputs_documented.py b/tests/integration/meta/workflows/test_ci_inputs_documented.py new file mode 100644 index 0000000000..3676d7d1aa --- /dev/null +++ b/tests/integration/meta/workflows/test_ci_inputs_documented.py @@ -0,0 +1,96 @@ +"""Integration guard: every repository secret and variable a workflow reads is documented. + +Background +========== +A `${{ secrets.X }}` or `${{ vars.X }}` reference is a contract with whoever +administers the repository, and an unset one resolves to the empty string +rather than to an error. An undocumented name is therefore invisible twice: the +administrator cannot know to set it, and the run that needed it degrades +silently. + +The guard also catches the inverse drift, a doc naming a variable the workflows +no longer read under that name, because it compares the reference against the +page rather than the page against itself. + +Scope +===== +Every `${{ secrets. }}` and `${{ vars. }}` on a non-comment line of +`.github/workflows/*.yml`. Secrets MUST appear in `secrets.md`, variables in +`configuration.md`, both under `docs/contributing/tools/github/actions/`. + +Exemptions +========== +`GITHUB_TOKEN` is minted by GitHub for every run; it is not administered and +cannot be set. Any other exemption belongs in `_PROVIDED_SECRETS` with the +reason it is not administrator-supplied. +""" + +from __future__ import annotations + +import re +import unittest + +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +WORKFLOWS = PROJECT_ROOT / ".github" / "workflows" +DOCS = PROJECT_ROOT / "docs" / "contributing" / "tools" / "github" / "actions" + +_REFERENCE = re.compile(r"\$\{\{\s*(secrets|vars)\.([A-Za-z_][A-Za-z0-9_]*)") + +_PROVIDED_SECRETS = frozenset({"GITHUB_TOKEN"}) + +_PAGES = {"secrets": "secrets.md", "vars": "configuration.md"} + + +def _referenced() -> dict[str, set[str]]: + """Names the workflows actually read, keyed by `secrets` / `vars`. + + A commented-out line is skipped: it documents an option nobody has taken, + and demanding a doc row for it would describe an input the run never asks + for. + """ + found: dict[str, set[str]] = {"secrets": set(), "vars": set()} + for workflow in sorted(WORKFLOWS.glob("*.yml")): + for line in read_text(str(workflow)).splitlines(): + if line.lstrip().startswith("#"): + continue + for kind, name in _REFERENCE.findall(line): + if kind == "secrets" and name in _PROVIDED_SECRETS: + continue + found[kind].add(name) + return found + + +class TestCiInputsDocumented(unittest.TestCase): + def test_every_referenced_secret_and_variable_is_documented(self) -> None: + offenders: list[str] = [] + for kind, names in _referenced().items(): + documented = read_text(str(DOCS / _PAGES[kind])) + offenders.extend( + f"{kind}.{name} is read by a workflow but absent from " + f"docs/contributing/tools/github/actions/{_PAGES[kind]}" + for name in sorted(names) + if name not in documented + ) + + if offenders: + self.fail( + f"{len(offenders)} CI input(s) undocumented:\n" + + "\n".join(f"- {o}" for o in offenders) + + "\n\nAn unset secret or variable resolves to the empty string, " + "so an undocumented one degrades a run instead of failing it. " + "Add a row naming the workflow, what it enables, and what " + "happens while it is unset." + ) + + def test_the_guard_reads_workflows_that_exist(self) -> None: + self.assertTrue( + any(WORKFLOWS.glob("*.yml")), + f"no workflow files under {WORKFLOWS}; the guard would pass vacuously", + ) + + +if __name__ == "__main__": + unittest.main() From 3b2761c15f6379655db537e9d21fbd0040823ee6 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 20:13:57 +0200 Subject: [PATCH 018/197] perf(deploy): run the per-host setup once per host, not once per application MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two probes that describe a host were re-running per application. Measured in CI run wp-swarm-tor2: 42 invocations of the pip probe and 44 of the NFS assert. sys-pip-install guarded only its dependency include. Everything expensive sat below that guard in main.yml, unguarded: resolving the interpreter, probing pip for --break-system-packages, checking for `packaging` and bootstrapping it. All four ran on every one of the 21 include sites. They move into 00_core.yml, where the existing `when: run_once_sys_pip_install is not defined` covers them, and main.yml keeps the guarded include and the pip task itself. The role recap reads sys-pip-install ---- 87.35s. The two facts this freezes per host are run constants: ansible_python_interpreter is pinned once in group_vars/all/00_general.yml and no call site overrides it, and all 21 sites pass global_install: true. main.yml now defends the probe fact with `| default(false)` the way 00_core.yml already did, because the probe sits behind `when: global_install | bool` and a first invocation with false would otherwise leave it undefined for every later one. The NFS assert emulated a per-host check with run_once plus delegate_to over groups['svc-swarm-node'], and the include it lives in happens per application. It becomes a plain per-host task in sys-svc-compose/00_core.yml, which main.yml already gates once per host per play: 44 instances and 132 remote commands become 4 and 4. Dropping run_once also removes a real fragility โ€” at log line 35355 only wrk-01 and wrk-02 reach the include because mgr-01's flag is already set, so run_once there designates a host that skipped. It reaches the same hosts as the loop it replaces, measured rather than assumed: sys-svc-compose/00_core fires only on mgr-01, wrk-01 and wrk-02, the backup play never runs it, and nfs-server and bkp-01 never execute it. The two conditions that do not travel are a per-application volume count, meaningless for a per-host check, and an exemption for manager-placed roles โ€” mgr-01 answered the assert ok 14 times out of 14, so the exemption was hiding nothing. The assert is appended to 00_core.yml rather than prepended: utils/once/flag.yml must stay that file's first task for test_service_core_first_task_run_once. Worth 128.6s: 84.6s from the pip probe and 44.0s from the assert. Co-Authored-By: Claude Opus 5 (1M context) --- roles/sys-pip-install/tasks/00_core.yml | 66 ++++++++++++++++ roles/sys-pip-install/tasks/main.yml | 78 ++----------------- roles/sys-svc-compose/tasks/00_core.yml | 8 ++ .../sys-svc-compose/tasks/utils/nfs_prep.yml | 13 ---- 4 files changed, 80 insertions(+), 85 deletions(-) diff --git a/roles/sys-pip-install/tasks/00_core.yml b/roles/sys-pip-install/tasks/00_core.yml index 65dd4be738..7c25a2fc10 100644 --- a/roles/sys-pip-install/tasks/00_core.yml +++ b/roles/sys-pip-install/tasks/00_core.yml @@ -5,3 +5,69 @@ when: run_once_sys_pip is not defined - include_tasks: utils/once/flag.yml + +- name: ๐Ÿ” Resolve active Python executable for sys-pip-install + ansible.builtin.set_fact: + sys_pip_install_python_executable: "{{ ansible_python_interpreter }}" + changed_when: false + +- block: + - name: ๐Ÿ” Probe pip for ansible_python_interpreter accepts --break-system-packages + ansible.builtin.command: + argv: + - "{{ sys_pip_install_python_executable }}" + - -m + - pip + - install + - --break-system-packages + - --help + register: sys_pip_install_break_system_packages_probe + changed_when: false + failed_when: false + become: "{{ true if global_install | bool else false }}" + + - name: ๐Ÿ”ง Set fact for --break-system-packages support + ansible.builtin.set_fact: + sys_pip_install_supports_break_system_packages: "{{ (sys_pip_install_break_system_packages_probe.rc | int) == 0 }}" + changed_when: false + + when: global_install | bool + +- name: ๐Ÿ”Ž Check whether selected Python interpreter has 'packaging' + ansible.builtin.command: + argv: + - "{{ sys_pip_install_python_executable }}" + - -I + - -c + - import packaging + register: sys_pip_install_python_packaging_check + changed_when: false + failed_when: false + +- name: ๐Ÿฅพ Bootstrap 'packaging' for selected Python interpreter + ansible.builtin.command: + argv: >- + {{ + [sys_pip_install_python_executable, '-m', 'pip', 'install', '--upgrade', 'packaging'] + + ( + ['--break-system-packages'] + if (global_install | bool and sys_pip_install_supports_break_system_packages | default(false) | bool) + else [] + ) + }} + environment: >- + {{ + ( + {'PIP_BREAK_SYSTEM_PACKAGES': '1'} + if sys_pip_install_supports_break_system_packages | default(false) | bool + else {} + ) + if global_install | bool + else omit + }} + become: "{{ true if global_install | bool else false }}" + register: sys_pip_install_packaging_bootstrap + retries: 5 + delay: 15 + until: sys_pip_install_packaging_bootstrap is succeeded + when: sys_pip_install_python_packaging_check.rc != 0 diff --git a/roles/sys-pip-install/tasks/main.yml b/roles/sys-pip-install/tasks/main.yml index 0709680650..415eae462e 100644 --- a/roles/sys-pip-install/tasks/main.yml +++ b/roles/sys-pip-install/tasks/main.yml @@ -4,55 +4,12 @@ - include_tasks: 00_core.yml when: run_once_sys_pip_install is not defined -- name: ๐Ÿ” Resolve active Python executable for sys-pip-install - ansible.builtin.set_fact: - sys_pip_install_python_executable: "{{ ansible_python_interpreter }}" - changed_when: false - -- block: - - name: ๐Ÿ” Probe pip for ansible_python_interpreter accepts --break-system-packages - ansible.builtin.command: - argv: - - "{{ sys_pip_install_python_executable }}" - - -m - - pip - - install - - --break-system-packages - - --help - register: sys_pip_install_break_system_packages_probe - changed_when: false - failed_when: false - become: "{{ true if global_install | bool else false }}" - - - name: ๐Ÿ”ง Set fact for --break-system-packages support - ansible.builtin.set_fact: - sys_pip_install_supports_break_system_packages: "{{ (sys_pip_install_break_system_packages_probe.rc | int) == 0 }}" - changed_when: false - - when: global_install | bool - -- name: ๐Ÿ”Ž Check whether selected Python interpreter has 'packaging' - ansible.builtin.command: - argv: - - "{{ sys_pip_install_python_executable }}" - - -I - - -c - - import packaging - register: sys_pip_install_python_packaging_check - changed_when: false - failed_when: false - -- name: ๐Ÿฅพ Bootstrap 'packaging' for selected Python interpreter - ansible.builtin.command: - argv: >- - {{ - [sys_pip_install_python_executable, '-m', 'pip', 'install', '--upgrade', 'packaging'] - + ( - ['--break-system-packages'] - if (global_install | bool and sys_pip_install_supports_break_system_packages | default(false) | bool) - else [] - ) - }} +- name: ๐Ÿ“ฆ Install or upgrade {{ package_name }} system-wide via pip + ansible.builtin.pip: + name: "{{ package_name }}" + state: "{{ package_state }}" + extra_args: "{{ '--break-system-packages' if (global_install | bool and sys_pip_install_supports_break_system_packages | default(false) | bool) else omit }}" + virtualenv: "{{ omit if global_install | bool else (sys_pip_install_python_executable | dirname | dirname) }}" environment: >- {{ ( @@ -64,29 +21,6 @@ else omit }} become: "{{ true if global_install | bool else false }}" - register: sys_pip_install_packaging_bootstrap - retries: 5 - delay: 15 - until: sys_pip_install_packaging_bootstrap is succeeded - when: sys_pip_install_python_packaging_check.rc != 0 - -- name: ๐Ÿ“ฆ Install or upgrade {{ package_name }} system-wide via pip - ansible.builtin.pip: - name: "{{ package_name }}" - state: "{{ package_state }}" - extra_args: "{{ '--break-system-packages' if (global_install | bool and sys_pip_install_supports_break_system_packages | bool) else omit }}" - virtualenv: "{{ omit if global_install | bool else (sys_pip_install_python_executable | dirname | dirname) }}" - environment: >- - {{ - ( - {'PIP_BREAK_SYSTEM_PACKAGES': '1'} - if sys_pip_install_supports_break_system_packages | bool - else {} - ) - if global_install | bool - else omit - }} - become: "{{ true if global_install | bool else false }}" register: sys_pip_install_result retries: 5 delay: 15 diff --git a/roles/sys-svc-compose/tasks/00_core.yml b/roles/sys-svc-compose/tasks/00_core.yml index c0d631675f..9e53702465 100644 --- a/roles/sys-svc-compose/tasks/00_core.yml +++ b/roles/sys-svc-compose/tasks/00_core.yml @@ -44,3 +44,11 @@ mode: "0755" owner: root group: root + +- name: "๐Ÿ›ก๏ธ Assert the NFS client mount is active on this swarm node" + ansible.builtin.command: "mountpoint -q {{ DIR_VAR_LIB }}" + changed_when: false + when: + - DEPLOYMENT_MODE == 'swarm' + - (storage.backend | default('local')) == 'nfs' + - (groups['svc-storage-nfs-server'] | default([])) | length > 0 diff --git a/roles/sys-svc-compose/tasks/utils/nfs_prep.yml b/roles/sys-svc-compose/tasks/utils/nfs_prep.yml index f22615d86d..f7f528da7b 100644 --- a/roles/sys-svc-compose/tasks/utils/nfs_prep.yml +++ b/roles/sys-svc-compose/tasks/utils/nfs_prep.yml @@ -19,19 +19,6 @@ - item.value.driver_opts is defined - application_id not in lookup('roles_with_placement', 'manager') -- name: "๐Ÿ›ก๏ธ Assert the NFS client mount is active on every swarm node" - ansible.builtin.command: "mountpoint -q {{ DIR_VAR_LIB }}" - changed_when: false - delegate_to: "{{ item }}" - run_once: true - loop: "{{ groups['svc-swarm-node'] | default([]) }}" - when: - - DEPLOYMENT_MODE == 'swarm' - - (storage.backend | default('local')) == 'nfs' - - (groups['svc-storage-nfs-server'] | default([])) | length > 0 - - (lookup('compose_volumes') | from_yaml).volumes | default({}) | length > 0 - - application_id not in lookup('roles_with_placement', 'manager') - - name: "๐Ÿ” Materialize NFS subdirs for '{{ application_id }}' on every swarm node" ansible.builtin.file: path: "{{ DIR_VAR_LIB }}/{{ item.0.value.name | default(item.0.key) }}" From 46110f4311a211e7742c9dd0e1894b34044b074a Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 20:32:02 +0200 Subject: [PATCH 019/197] perf(swarm-test): rotate the round's credentials, not every block in the file The matrix gate rotates credentials between the two deploy passes so PASS 2 has to carry fresh values. It rotated every application block in the host_vars file, and on a matrix inventory that is not the round. Provision applies its --include as a strict intersection (filters.py:57-62), so it generates credentials for at most derive_includes, while mirror_overrides.py adds a block per mirror artefact carrying only image and version. Rotating those costs one credential subprocess each for values provision never wrote. matrix.py now passes --include derive_includes(app_id, variants=round_variants), the same call 02_provision_inventory.sh:11 feeds provision, from the same round_variants the provision step got one call earlier. The rotation set is therefore a superset of what provision generated: every database_password still rotates, because utils/manager/inventory.py:236-242 writes it under an id from that same filtered list. Measured for web-app-wordpress: 35 ids. The scope stops at the credentials. reset_credentials takes credential_ids beside application_ids because the two halves shared one list, and narrowing it narrowed _drop_user_passwords too: that resolves through required_user_policies(roles_dir, application_ids), so 60 of 66 declared users would have stopped rotating, root, backup and every mcp-* bot among them. The user half keeps the full document list and is byte-identical to before. The test fixture declared the same users in both roles, which made the regression test pass against the regression. web-app-keycloak now declares a user web-app-a does not, so the assertion has something to bite on. The CLI aborts when --include matches no block instead of rotating nothing. Co-Authored-By: Claude Opus 5 (1M context) --- .../inventory/credentials/reset/cli.py | 24 +++++++++- .../inventory/provision/reset.py | 18 ++++++-- .../inventory/provision/test_reset.py | 45 +++++++++++++++++-- utils/tests/swarm/matrix.py | 18 +++++++- 4 files changed, 96 insertions(+), 9 deletions(-) diff --git a/cli/administration/inventory/credentials/reset/cli.py b/cli/administration/inventory/credentials/reset/cli.py index 0d901aec54..9287c30b65 100644 --- a/cli/administration/inventory/credentials/reset/cli.py +++ b/cli/administration/inventory/credentials/reset/cli.py @@ -119,6 +119,18 @@ def main(argv: list[str] | None = None) -> int: default=[], help="Application ids and user keys to leave untouched (e.g. administrator).", ) + parser.add_argument( + "--include", + nargs="*", + default=None, + help=( + "Application ids whose credentials rotate. Absent rotates every " + "application block in the file, which on a matrix inventory is " + "every mirror artefact too, not just the ids the round " + "provisioned. User passwords ignore this and always rotate in " + "full." + ), + ) parser.add_argument( "--app-variants", default=None, @@ -161,16 +173,26 @@ def main(argv: list[str] | None = None) -> int: if not application_ids: raise SystemExit(f"No applications block to rotate in {host_vars_file}") + credential_ids = None + if args.include is not None: + wanted = set(args.include) + credential_ids = [app_id for app_id in application_ids if app_id in wanted] + if not credential_ids: + raise SystemExit(f"--include matches no block in {host_vars_file}") + if args.backup: print(f"[INFO] Pre-rotation copy: {_backup(host_vars_file)}") exclude = set(args.exclude) print( f"[INFO] Rotating credentials in {host_vars_file} " - f"({len(application_ids)} applications, excluding {sorted(exclude) or 'nothing'})" + f"({len(credential_ids or application_ids)} applications, " + f"{len(application_ids)} for users, " + f"excluding {sorted(exclude) or 'nothing'})" ) rotated = reset_credentials( application_ids=application_ids, + credential_ids=credential_ids, roles_dir=(project_root / "roles").resolve(), host_vars_file=host_vars_file, vault_password_file=vault_password_file, diff --git a/cli/administration/inventory/provision/reset.py b/cli/administration/inventory/provision/reset.py index 19f24de29c..54e5bfba82 100644 --- a/cli/administration/inventory/provision/reset.py +++ b/cli/administration/inventory/provision/reset.py @@ -117,13 +117,17 @@ def _drop_app_credentials( document: CommentedMap, exclude: set[str], pinned: dict[str, set[tuple[str, ...]]], + application_ids: list[str], ) -> int: - """Remove the generated credentials of every application in ``document``. + """Remove the generated credentials of the applications being rotated. Args: document: the host_vars document. exclude: application ids to leave untouched. pinned: per-application credential paths that must survive a rotation. + application_ids: the ids the caller regenerates afterwards. Passing a + narrower set here than the caller regenerates would drop + credentials nothing writes back. Returns: How many values were removed. @@ -132,10 +136,13 @@ def _drop_app_credentials( if not isinstance(applications, CommentedMap): return 0 + rotated = set(application_ids) dropped = 0 for application_id, application in applications.items(): if application_id in exclude or not isinstance(application, CommentedMap): continue + if str(application_id) not in rotated: + continue secrets = application.get(SECRETS_KEY) if not isinstance(secrets, CommentedMap): continue @@ -194,6 +201,7 @@ def reset_credentials( exclude: set[str], workers: int = 4, app_variants: dict[str, int] | None = None, + credential_ids: list[str] | None = None, ) -> int: """Replace every generated credential in ``host_vars_file`` with a fresh one. @@ -209,6 +217,9 @@ def reset_credentials( exclude: application ids and user keys to leave untouched. workers: worker threads for credentials generation. app_variants: variant index per app, as at provision time. + credential_ids: application ids whose credentials rotate; ``None`` + rotates all of ``application_ids``. User passwords ignore it and + always resolve through the full ``application_ids``. Returns: How many values were rotated. @@ -216,11 +227,12 @@ def reset_credentials( if not schema and not users: raise SystemExit("reset_credentials: pick at least one of schema, users") + rotated_ids = application_ids if credential_ids is None else credential_ids document = load_document(host_vars_file) dropped = 0 if schema: dropped += _drop_app_credentials( - document, exclude, _pinned_credentials(roles_dir) + document, exclude, _pinned_credentials(roles_dir), rotated_ids ) if users: dropped += _drop_user_passwords(document, roles_dir, application_ids, exclude) @@ -228,7 +240,7 @@ def reset_credentials( if schema: generate_credentials_for_roles( - application_ids=application_ids, + application_ids=rotated_ids, roles_dir=roles_dir, host_vars_file=host_vars_file, vault_password_file=vault_password_file, diff --git a/tests/unit/python/cli/administration/inventory/provision/test_reset.py b/tests/unit/python/cli/administration/inventory/provision/test_reset.py index 91d038ee8f..ae19201f32 100644 --- a/tests/unit/python/cli/administration/inventory/provision/test_reset.py +++ b/tests/unit/python/cli/administration/inventory/provision/test_reset.py @@ -53,6 +53,10 @@ users: 6e6577730a biber: password: VeryUnsecurePassword!1 + keycloak-bot: + password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 626f740a """ DECLARED_USERS = """\ @@ -63,6 +67,11 @@ newsletter: uid: 1002 """ +KEYCLOAK_USERS = DECLARED_USERS + """\ +keycloak-bot: + uid: 1003 +""" + class TestResetCredentials(unittest.TestCase): def setUp(self): @@ -70,10 +79,14 @@ class TestResetCredentials(unittest.TestCase): self.addCleanup(shutil.rmtree, self.workdir, ignore_errors=True) self.roles_dir = self.workdir / "roles" - for application_id in ("web-app-a", "web-app-keycloak"): + declarations = { + "web-app-a": DECLARED_USERS, + "web-app-keycloak": KEYCLOAK_USERS, + } + for application_id, declared in declarations.items(): users_file = self.roles_dir / application_id / ROLE_FILE_META_USERS users_file.parent.mkdir(parents=True, exist_ok=True) - users_file.write_text(DECLARED_USERS, encoding="utf-8") + users_file.write_text(declared, encoding="utf-8") self.host_vars_file = self.workdir / "host_vars" / "localhost.yml" self.host_vars_file.parent.mkdir(parents=True, exist_ok=True) @@ -125,6 +138,32 @@ class TestResetCredentials(unittest.TestCase): self._reset(exclude={"administrator", "web-app-keycloak"}) self.assertIn("database_password", self._credentials("web-app-keycloak")) + def test_an_application_outside_the_rotation_keeps_its_credentials(self): + """Dropping must not outrun regeneration. + + The drop side walks the document while the generate side walks + `credential_ids`. A matrix host_vars file carries a block per mirror + artefact too, so a document-wide drop would delete credentials the + caller never writes back. + """ + _, credentials, _ = self._reset(credential_ids=["web-app-a"]) + self.assertIn("database_password", self._credentials("web-app-keycloak")) + self.assertNotIn("api_key", self._credentials("web-app-a")) + self.assertEqual(credentials.call_args.kwargs["application_ids"], ["web-app-a"]) + + def test_narrowing_the_credentials_still_rotates_every_user(self): + """A narrowed credential scope must not narrow the user passwords. + + Both halves used to share one id list, so scoping the credentials to + the round silently stopped rotating the users of every other role. + """ + _, _, users = self._reset(credential_ids=["web-app-a"]) + self.assertNotIn("password", self._document()["users"]["keycloak-bot"]) + self.assertEqual( + users.call_args.kwargs["application_ids"], + ["web-app-a", "web-app-keycloak"], + ) + def test_an_excluded_user_keeps_its_password(self): self._reset() users = self._document()["users"] @@ -164,7 +203,7 @@ class TestResetCredentials(unittest.TestCase): def test_the_rotated_count_covers_both_sections(self): rotated, _, _ = self._reset() - self.assertEqual(rotated, 4) + self.assertEqual(rotated, 5) def test_rotating_nothing_aborts(self): with self.assertRaises(SystemExit): diff --git a/utils/tests/swarm/matrix.py b/utils/tests/swarm/matrix.py index c75ac88320..3edf062f6b 100644 --- a/utils/tests/swarm/matrix.py +++ b/utils/tests/swarm/matrix.py @@ -29,6 +29,7 @@ import time from utils import PROJECT_ROOT from utils.env.runtime import mem_available_mb, mem_stall_pct, mem_total_mb from utils.storage.constrained import host_storage_constrained +from utils.tests.swarm.derive_includes import derive_includes from utils.tests.swarm.write.extras import ensure_swarm_keypairs _SWARM_DIR = PROJECT_ROOT / "scripts" / "tests" / "deploy" / "swarm" @@ -167,11 +168,20 @@ def _write_extras(*, extras_path: str) -> int: ) -def _reset_credentials(*, inv_dir: str, round_variants: dict[str, int]) -> int: +def _reset_credentials( + *, app_id: str, inv_dir: str, round_variants: dict[str, int] +) -> int: """Regenerate the round's credentials so the update pass has to carry them. `administrator` stays exempt: its password is `ansible_become_password`, and rotating it would lock the deploy out of the nodes it manages. + + The credential scope is `derive_includes`, the same source + `02_provision_inventory.sh` feeds provision's `--include`, so the gate + rotates the ids the round provisioned. A matrix host_vars file also holds + an application block per mirror artefact, and rotating those costs one + subprocess each for credentials provision never generated. Every declared + user password still rotates, so PASS 2 has to carry all of them. """ return _run( [ @@ -184,6 +194,8 @@ def _reset_credentials(*, inv_dir: str, round_variants: dict[str, int]) -> int: os.environ["MGR"], "--schema", "--users", + "--include", + *derive_includes(app_id, variants=round_variants), "--app-variants", json.dumps(round_variants, sort_keys=True), "--mirror", @@ -452,7 +464,9 @@ def main(argv: list[str] | None = None) -> int: app_id=app_id, inv_dir=inv_root, extras_path=extras_path ) if rc == 0: - rc = _reset_credentials(inv_dir=inv_root, round_variants=round_variants) + rc = _reset_credentials( + app_id=app_id, inv_dir=inv_root, round_variants=round_variants + ) if rc == 0: rc = _deploy( app_id=app_id, From b356954d2e0382363e147b5e985b861a951756f6 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 20:50:27 +0200 Subject: [PATCH 020/197] style(inventory): wrap the keycloak users fixture the way autoformat does make quality-high pass 75 rewrote it: a concatenation whose right operand is a multi-line literal gets parenthesised, with the operator leading its own line. The string itself is unchanged, so test_narrowing_the_credentials_still_rotates_every_user keeps the declaration that separates web-app-keycloak from web-app-a and still fails when the user half of the rotation is narrowed along with the credentials. Co-Authored-By: Claude Opus 5 (1M context) --- .../cli/administration/inventory/provision/test_reset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/python/cli/administration/inventory/provision/test_reset.py b/tests/unit/python/cli/administration/inventory/provision/test_reset.py index ae19201f32..4995e6059f 100644 --- a/tests/unit/python/cli/administration/inventory/provision/test_reset.py +++ b/tests/unit/python/cli/administration/inventory/provision/test_reset.py @@ -67,10 +67,13 @@ newsletter: uid: 1002 """ -KEYCLOAK_USERS = DECLARED_USERS + """\ +KEYCLOAK_USERS = ( + DECLARED_USERS + + """\ keycloak-bot: uid: 1003 """ +) class TestResetCredentials(unittest.TestCase): From 5097ccdfe3aa1e6a21193872f02dadde1b312dcf Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 14 Sep 2026 22:29:59 +0200 Subject: [PATCH 021/197] fix(inventory): stop holding a default's template text to the value's pattern Run 34883491022 lost all 15 of its jobs and run 34883296966 four more, every one of them on the same line and before any role deployed: ERROR: the value supplied for 'credentials.openai_api_key' does not match the pattern its schema declares: \S{20,} 786899700f gave the three provider keys a regex and wired validation onto the value that sits beside a `default:`. That value is the default itself. The credential leaf writes meta["default"] into the inventory verbatim, so the next read hands `{{ API.openai.api_key }}` back as if an operator had typed it. The literal is 24 characters and carries spaces, so \S{20,} cannot match it, and svc-ai-litellm sits in every closure that holds a litellm consumer, which is why roles as distant as Baserow and Memcached died with it. A pattern describes the rendered secret. Template text is not that value, so validate_supplied_value returns on it, next to the guard that already returns on an empty one. One guard covers all three call sites. It can only skip a check, never add one, and the three roles carrying a Jinja default for far longer (listmonk, keycloak, espocrm) declare no regex and keep their behaviour. The test class asserted that `` is rejected but never that the schema's own default survives, so the gap shipped green. test_every_declared_default_survives_its_own_schema closes it for every default the file declares, which is why _schema() widened into _credentials(). Reproduced at the call site that failed: the credentials subprocess taken from the job log exits 0, and the value it emits decrypts back to the template. Co-Authored-By: Claude Opus 5 (1M context) --- .../utils/manager/test_inventory_manager.py | 24 +++++++++++++++++-- utils/manager/inventory.py | 10 ++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/tests/unit/python/utils/manager/test_inventory_manager.py b/tests/unit/python/utils/manager/test_inventory_manager.py index 19028fa34c..96a98c2b60 100644 --- a/tests/unit/python/utils/manager/test_inventory_manager.py +++ b/tests/unit/python/utils/manager/test_inventory_manager.py @@ -802,6 +802,11 @@ class TestSuppliedValueValidation(TestCase): def test_a_schema_declaring_neither_accepts_anything(self): validate_supplied_value("credentials.x", "anything", {"description": "free"}) + def test_an_unrendered_template_is_not_a_value(self): + for value in ("{{ API.openai.api_key }}", "{% if x %}a{% endif %}"): + with self.subTest(value=value): + validate_supplied_value("credentials.api_key", value, self.SCHEMA) + class TestProviderKeyPattern(TestCase): """The shipped provider-key pattern must outlive a vendor's format change. @@ -825,9 +830,24 @@ class TestProviderKeyPattern(TestCase): "sk-proj-Ab Cd 1234567890 xyz", ) - def _schema(self) -> dict: + def _credentials(self) -> dict: path = PROJECT_ROOT / "roles/svc-ai-litellm" / ROLE_FILE_META_SECRETS - return load_yaml_any(str(path))["credentials"]["openai_api_key"] + return load_yaml_any(str(path))["credentials"] + + def _schema(self) -> dict: + return self._credentials()["openai_api_key"] + + def test_every_declared_default_survives_its_own_schema(self): + """A default is written verbatim and read back as a supplied value. + + Run 34883491022 died in every job because the provider keys' own + `{{ API..api_key }}` reached the pattern on the second read. + """ + for key, schema in self._credentials().items(): + if "default" not in schema: + continue + with self.subTest(key=key): + validate_supplied_value(f"credentials.{key}", schema["default"], schema) def test_every_known_provider_form_is_accepted(self): schema = self._schema() diff --git a/utils/manager/inventory.py b/utils/manager/inventory.py index 94d0907cbf..2c7379d022 100644 --- a/utils/manager/inventory.py +++ b/utils/manager/inventory.py @@ -35,6 +35,8 @@ _SCHEMA_TYPES: dict[str, type | tuple[type, ...]] = { "boolean": bool, } +_UNRENDERED_TEMPLATE = re.compile(r"\{\{|\{%") + def _is_credential_leaf(node: Any) -> bool: return isinstance(node, dict) and any( @@ -55,10 +57,18 @@ def validate_supplied_value(full_key: str, value: Any, meta: dict[str, Any]) -> Raises: SystemExit: the value contradicts the declared type or pattern. + + Exception: a `default:` is written into the inventory verbatim, so on the + next read it arrives here as if an operator had typed it. Every default in + the repository is a Jinja reference, and a pattern describes the rendered + secret, not the template text that will produce it. """ if value in ("", None): return + if isinstance(value, str) and _UNRENDERED_TEMPLATE.search(value): + return + declared = meta.get("type") if declared: expected = _SCHEMA_TYPES.get(str(declared)) From f8da0fd12259e395966b108d48f276ef0e921fee Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 00:50:45 +0200 Subject: [PATCH 022/197] fix(ai): pin each LM Studio model to its exact source, not to a search term Run 34893750601 lost four jobs at svc-ai-litellm's warm request, every one of them on the same upstream body: No models loaded. Please load a model in the developer page or use the `lms load` command. The catalogue handed `lms get --yes --gguf` a bare name, which that command treats as a hub search term. Its own help spells out the consequence: "If there are multiple staff picks matching the search term, the first one will be used." For `qwen2.5-0.5b-instruct` the first pick was Qwen2.5-Coder-0.5B-Instruct, so the key LM Studio indexed never matched the `openai/{{ _model.name }}` the gateway sends upstream, and just-in-time loading had nothing to load. A `source:` now carries the full HuggingFace URL, the one spelling lms get resolves deterministically; a bare owner/model pair addresses an LM Studio Hub artifact instead and fails even for a model already on disk. Measured against the shipped image rather than reasoned: the URL yields modelKey qwen2.5-0.5b-instruct, `lms ps` is empty right before the request, and a plain /v1/chat/completions for that name answers 200 and leaves the model resident. No load step belongs here, because the server reports "Just-in-time model loading active" and does it itself. The embedding entry is renamed in the same breath. LM Studio indexes that model as text-embedding-nomic-embed-text-v1.5, so the gateway was addressing a key that never existed there either, and the first consumer to embed would have hit the same 404. The three fields stay separate and each has exactly one reader: Ollama pulls by alias, LM Studio downloads by source, the gateway addresses name. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/16_ai.yml | 5 ++++- inventories/development/default.yml | 1 + roles/svc-ai-lmstudio/tasks/01_manager_ops.yml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index d68976e3c0..23602bfe69 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -16,10 +16,13 @@ LITELLM_BACKEND_ROLES: "{{ lookup('deployment').inventory }}" AI_LOCAL_MODELS: - alias: llama3:latest name: llama-3.2-3b-instruct + source: https://huggingface.co/lmstudio-community/Llama-3.2-3B-Instruct-GGUF - alias: mistral:latest name: mistral-7b-instruct-v0.3 + source: https://huggingface.co/lmstudio-community/Mistral-7B-Instruct-v0.3-GGUF - alias: nomic-embed-text:latest - name: nomic-embed-text-v1.5 + name: text-embedding-nomic-embed-text-v1.5 + source: https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF AI_REMOTE_ALIASES: >- {{ ['openai/gpt-4o-mini'] * ((API.openai.api_key | string | length > 0) | int) diff --git a/inventories/development/default.yml b/inventories/development/default.yml index eb5f74116e..09aa0a54dd 100644 --- a/inventories/development/default.yml +++ b/inventories/development/default.yml @@ -38,6 +38,7 @@ design: AI_LOCAL_MODELS_CI: - alias: qwen2.5:0.5b name: qwen2.5-0.5b-instruct + source: https://huggingface.co/lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF API: openai: diff --git a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml index d667dca141..60f97c71e8 100644 --- a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml +++ b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml @@ -8,7 +8,7 @@ - name: โฌ‡๏ธ Pre-pull LM Studio models ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node - container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} lms get --yes --gguf {{ model.name | quote }} + container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} lms get --yes --gguf {{ model.source | quote }} register: lmstudio_pull_result loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" loop_control: From ea2e373e1cd8e7b1ab2784c375d247d6b22ba435 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 00:56:01 +0200 Subject: [PATCH 023/197] fix(discourse): only set the admin password when it is not already the one set Job 104153193860 died at "Ensure the administrator account carries the current password" with: Validation failed: Password is the same as your current password. Discourse's UserPasswordValidator adds :same_as_current when password_hash_ changed? and confirm_password?(value) are both true. Assigning the current password again satisfies both, because the reassignment salts a fresh hash. The script assigned unconditionally, so any run that did not follow a rotation was a hard failure rather than a no-op, and the task was never idempotent. The assignment is now guarded by confirm_password?, which makes the rule unreachable: it requires confirm_password? to be true, and that is exactly when nothing is assigned. A new user is covered without a new_record? check, since User#confirm_password? returns false when no user_password row exists. web-app-openproject/files/ruby/ensure_admin_user.rb already guards the same way with check_password?; this brings Discourse in line with it. The two saves collapse into one while we are here. The second existed only to set `active`, and one save writes username, password and active together. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-discourse/files/ruby/ensure_admin_user.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/roles/web-app-discourse/files/ruby/ensure_admin_user.rb b/roles/web-app-discourse/files/ruby/ensure_admin_user.rb index 553ed9e5c8..15bb55151e 100644 --- a/roles/web-app-discourse/files/ruby/ensure_admin_user.rb +++ b/roles/web-app-discourse/files/ruby/ensure_admin_user.rb @@ -7,9 +7,7 @@ password = ENV.fetch("DISCOURSE_ADMIN_PASSWORD") user = User.find_by_email(email) || User.new(email: email) user.username = username -user.password = password -user.save! - +user.password = password unless user.confirm_password?(password) user.active = true user.save! From 691911279fd0e2857eb778d20cbebfb5e4999cbf Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 00:58:37 +0200 Subject: [PATCH 024/197] chore(tor): move the onion service from alpha to beta lifecycle is a selection axis rather than a label: cli/meta/ci/validate.py hands --lifecycles to matrix.entries_of, so a run whose envelope admits beta but not alpha picks svc-net-tor up from now on. Recent sweeps ran with an empty envelope and are unaffected. The maturity call is the operator's; this commit only records it. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-net-tor/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/svc-net-tor/meta/services.yml b/roles/svc-net-tor/meta/services.yml index 0b3345feac..5ffe165550 100644 --- a/roles/svc-net-tor/meta/services.yml +++ b/roles/svc-net-tor/meta/services.yml @@ -39,4 +39,4 @@ tor: # nocheck: category-modes mem_reservation: 128m mem_limit: 256m pids_limit: 128 - lifecycle: alpha + lifecycle: beta From b18710e1883e6a10d9a0a91ae80dc0b5e2a868bb Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 02:06:38 +0200 Subject: [PATCH 025/197] fix(litellm): publish a backend only when the gateway's own flag enables it Job 104155149606 died at the warm request with litellm.APIConnectionError: OllamaException - Cannot connect to host ollama:11434 [Connect call failed ('192.168.244.10', 11434)] 192.168.244.10 is not ollama. It is the manager node on the lab bridge: the name never resolved through the overlay, because the litellm task was attached to ingress, litellm and postgres and to nothing else. The lmstudio round is the control, where the same replica does carry the lmstudio network. The attachment comes from _is_consumer, which reads svc-ai-litellm's own services..enabled and .shared. Variant 1 pins both false. The config template meanwhile asked `'svc-ai-ollama' in group_names`, which answers a different question: is ollama deployed anywhere in the round. It is, six edges away, because litellm pulls postgres, postgres pulls tor, tor pulls the mirror, the mirror pulls prometheus and prometheus declares flowise, which declares ollama. So the gateway advertised a route it had no network for. The template and LITELLM_CHAT_MODEL_SERVED now read the same two flags the network attachment reads, through one constant each. Publication and routing cannot diverge any more, and in a variant that declines a deployed backend the warm task no longer runs at all. LITELLM_CHAT_MODEL and LITELLM_BACKEND_ROLES are deliberately left alone. 5324eadd8b made them inventory-scoped because web-app-mattermost's assert, the one task in its file carrying neither delegate_to nor run_once, re-templates in each worker's own scope; web-app-discourse's addon tasks carry no delegate_to at all. A host-scoped model name fails there again. Both templates here render delegate_to STACK_HOST plus run_once, where group_names is authoritative. meta/services.yml keeps its `in group_names` flags, so the five separate substring detectors that decide whether a flag counts as dynamic keep seeing svc-ai-litellm. Measured over every (deployed, pinned) combination against the real expressions and the real template: the model consumers are told to ask for is published whenever the warm task runs, and nothing is published without its flag. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/16_ai.yml | 12 ++++++++++-- roles/svc-ai-litellm/templates/config.yaml.j2 | 4 ++-- roles/svc-ai-litellm/templates/test.env.j2 | 8 ++++---- .../python/group_vars/test_litellm_chat_model.py | 9 +++++++++ .../templates/test_config_model_list.py | 15 +++++++++------ 5 files changed, 34 insertions(+), 14 deletions(-) diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index 23602bfe69..8191692603 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -13,6 +13,14 @@ LITELLM_OPENAI_BASE_LOCAL_URL: "{{ LITELLM_BASE_LOCAL_URL }}/v1" LITELLM_BACKEND_ROLES: "{{ lookup('deployment').inventory }}" +LITELLM_OLLAMA_BACKEND: >- + {{ (lookup('config', 'svc-ai-litellm', 'services.ollama.enabled') | bool) + and (lookup('config', 'svc-ai-litellm', 'services.ollama.shared') | bool) }} + +LITELLM_LMSTUDIO_BACKEND: >- + {{ (lookup('config', 'svc-ai-litellm', 'services.lmstudio.enabled') | bool) + and (lookup('config', 'svc-ai-litellm', 'services.lmstudio.shared') | bool) }} + AI_LOCAL_MODELS: - alias: llama3:latest name: llama-3.2-3b-instruct @@ -34,10 +42,10 @@ LITELLM_CHAT_MODEL: "{{ (lookup('config', 'svc-ai-ollama', 'services.ollama.prel | reject('search', 'embed') | list | first | default('')) if 'svc-ai-lmstudio' in LITELLM_BACKEND_ROLES else (AI_REMOTE_ALIASES | first | default(''))) }}" LITELLM_CHAT_MODEL_SERVED: >- - {{ (('svc-ai-ollama' in LITELLM_BACKEND_ROLES) + {{ ((LITELLM_OLLAMA_BACKEND | bool) and ((lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') | map(attribute='alias') | reject('search', 'embed') | list) | length > 0)) - or (('svc-ai-lmstudio' in LITELLM_BACKEND_ROLES) + or ((LITELLM_LMSTUDIO_BACKEND | bool) and ((lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') | map(attribute='alias') | reject('search', 'embed') | list) | length > 0)) or ((AI_REMOTE_ALIASES | length) > 0) }} diff --git a/roles/svc-ai-litellm/templates/config.yaml.j2 b/roles/svc-ai-litellm/templates/config.yaml.j2 index 6d7d02a8be..c4bed5dcce 100644 --- a/roles/svc-ai-litellm/templates/config.yaml.j2 +++ b/roles/svc-ai-litellm/templates/config.yaml.j2 @@ -1,5 +1,5 @@ -{% set _ollama_models = lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') if 'svc-ai-ollama' in group_names else [] %} -{% set _lmstudio_models = lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') if 'svc-ai-lmstudio' in group_names else [] %} +{% set _ollama_models = lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') if LITELLM_OLLAMA_BACKEND | bool else [] %} +{% set _lmstudio_models = lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') if LITELLM_LMSTUDIO_BACKEND | bool else [] %} {% set _remote_models = LITELLM_REMOTE_BACKENDS | selectattr('key') | list %} {% set has_backend = _ollama_models or _lmstudio_models or _remote_models %} model_list:{{ ' []' if not has_backend else '' }} diff --git a/roles/svc-ai-litellm/templates/test.env.j2 b/roles/svc-ai-litellm/templates/test.env.j2 index 47d7508fe7..42b48bb3c7 100644 --- a/roles/svc-ai-litellm/templates/test.env.j2 +++ b/roles/svc-ai-litellm/templates/test.env.j2 @@ -1,15 +1,15 @@ -{%- set _ollama_models = (lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') | map(attribute='alias') | list) if 'svc-ai-ollama' in group_names else [] -%} +{%- set _ollama_models = (lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') | map(attribute='alias') | list) if LITELLM_OLLAMA_BACKEND | bool else [] -%} {%- set _lmstudio_declared = lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') | map(attribute='alias') | list -%} {%- set _lmstudio_only = _lmstudio_declared | reject('in', _ollama_models) | list -%} {%- set _remote = LITELLM_REMOTE_BACKENDS | selectattr('key') | map(attribute='alias') | list -%} -{%- set _expected = _ollama_models + (_lmstudio_only if 'svc-ai-lmstudio' in group_names else []) + _remote -%} +{%- set _expected = _ollama_models + (_lmstudio_only if LITELLM_LMSTUDIO_BACKEND | bool else []) + _remote -%} LITELLM_CONTAINER={{ CLI_LOCAL_CID }} LITELLM_PORT={{ LITELLM_PORT }} LITELLM_MASTER_KEY_B64={{ LITELLM_MASTER_KEY | b64encode }} LITELLM_CHAT_MODEL={{ LITELLM_CHAT_MODEL }} LITELLM_CHAT_MODEL_SERVED={{ LITELLM_CHAT_MODEL_SERVED | bool | string | lower }} -LITELLM_OLLAMA_ENABLED={{ ('svc-ai-ollama' in group_names) | string | lower }} -LITELLM_LMSTUDIO_ENABLED={{ ('svc-ai-lmstudio' in group_names) | string | lower }} +LITELLM_OLLAMA_ENABLED={{ LITELLM_OLLAMA_BACKEND | bool | string | lower }} +LITELLM_LMSTUDIO_ENABLED={{ LITELLM_LMSTUDIO_BACKEND | bool | string | lower }} LITELLM_LMSTUDIO_ALIASES='{{ _lmstudio_only | to_json }}' LITELLM_REMOTE_ALIASES='{{ _remote | to_json }}' LITELLM_EXPECTED_MODELS='{{ _expected | to_json }}' diff --git a/tests/unit/python/group_vars/test_litellm_chat_model.py b/tests/unit/python/group_vars/test_litellm_chat_model.py index a348fb6268..67a8c4c902 100644 --- a/tests/unit/python/group_vars/test_litellm_chat_model.py +++ b/tests/unit/python/group_vars/test_litellm_chat_model.py @@ -11,6 +11,12 @@ from utils.cache.yaml import load_yaml _AI_VARS = Path(PROJECT_ROOT) / "group_vars" / "all" / "16_ai.yml" +def _ansible_bool(value): + if isinstance(value, str): + return value.strip().lower() in ("true", "yes", "on", "1") + return bool(value) + + def _stub_lookup(preload_models, lmstudio_models): def lookup(kind, role, path, *args): if (kind, path) == ("config", "services.ollama.preload_models"): @@ -38,6 +44,7 @@ class TestLitellmChatModel(unittest.TestCase): cls.env.tests["search"] = lambda value, pattern: bool( re.search(pattern, str(value)) ) + cls.env.filters["bool"] = _ansible_bool cls.source = load_yaml(_AI_VARS) def _render(self, name, *, roles, api_key, preload_models=(), lmstudio_models=()): @@ -45,6 +52,8 @@ class TestLitellmChatModel(unittest.TestCase): self.env.from_string(self.source[name]) .render( LITELLM_BACKEND_ROLES=list(roles), + LITELLM_OLLAMA_BACKEND=str("svc-ai-ollama" in roles), + LITELLM_LMSTUDIO_BACKEND=str("svc-ai-lmstudio" in roles), AI_REMOTE_ALIASES=(["openrouter/auto"] if api_key else []), lookup=_stub_lookup( [{"alias": alias, "name": alias} for alias in preload_models], diff --git a/tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py b/tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py index ef5fd8c093..e1728d0e72 100644 --- a/tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py +++ b/tests/unit/python/roles/svc-ai-litellm/templates/test_config_model_list.py @@ -28,6 +28,12 @@ OLLAMA_URL = "http://ollama:11434" LMSTUDIO_URL = "http://lmstudio:1234" +def _ansible_bool(value): + if isinstance(value, str): + return value.strip().lower() in ("true", "yes", "on", "1") + return bool(value) + + def _stub_lookup(ollama_models, lmstudio_models): def lookup(kind, role, path, *args): if (kind, path) == ("config", "services.ollama.preload_models"): @@ -66,14 +72,11 @@ def render(*, ollama=(), lmstudio=(), keys=None): lmstudio: preload entries svc-ai-lmstudio declares; empty means not deployed. keys: env-name -> provider key for the remote backends. """ - group_names = ["svc-ai-litellm"] - if ollama: - group_names.append("svc-ai-ollama") - if lmstudio: - group_names.append("svc-ai-lmstudio") env = Environment(undefined=StrictUndefined, autoescape=False) # noqa: S701 - YAML, not markup + env.filters["bool"] = _ansible_bool rendered = env.from_string(read_text(str(TEMPLATE))).render( - group_names=group_names, + LITELLM_OLLAMA_BACKEND=str(bool(ollama)), + LITELLM_LMSTUDIO_BACKEND=str(bool(lmstudio)), lookup=_stub_lookup(list(ollama), list(lmstudio)), OLLAMA_BASE_LOCAL_URL=OLLAMA_URL, LMSTUDIO_BASE_LOCAL_URL=LMSTUDIO_URL, From d7eb59e0516af734677a9f69b9d8e6588de2464a Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 02:23:55 +0200 Subject: [PATCH 026/197] fix(litellm): freeze the CLI test's routing contract where the stack renders Job 104155149593 failed its own CLI test with [FAIL] svc-ai-ollama is not deployed yet the gateway publishes ['qwen2.5:0.5b'] In that round ollama is deployed and the variant enables it, so the gateway was right and the probe was wrong. templates/test.env.j2 renders through lookup('template') in run_one.yml, which evaluates in the current play host's scope, while config.yaml.j2 renders through _render_one_template.yml with delegate_to STACK_HOST and run_once. On a worker the backend flags read false and the probe compared its own view against a gateway built from another one. The three flag-derived values are now frozen next to the config render, with the same delegate_to plus run_once. Which host run_once elects does not matter; what matters is that it is the same host for both, and that follows from the same construct in the same play. CLI_LOCAL_CID stays per host. 9fb8f63286 introduced it precisely because the container lives on whichever node the service scheduled it to, and the test script task carries no run_once, so it runs once per play host against its own resolved node. Freezing the render itself would have pinned one node's container id into every host's env and broken the CLI test for every role that execs into its own container, not just this one. Two facts this rests on, both measured rather than assumed: a run_once set_fact does reach the other hosts, as web-app-gitea's mcp provisioning shows by asserting on gitea_mcp_token from a task that carries no run_once; and 03_destructor.yml includes test-e2e-cli as a stage of the same play the deploy ran in, so the fact is still there when the template renders. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-litellm/tasks/00_core.yml | 9 +++++++++ roles/svc-ai-litellm/templates/test.env.j2 | 10 +++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/roles/svc-ai-litellm/tasks/00_core.yml b/roles/svc-ai-litellm/tasks/00_core.yml index db70913e93..9076c123b0 100644 --- a/roles/svc-ai-litellm/tasks/00_core.yml +++ b/roles/svc-ai-litellm/tasks/00_core.yml @@ -21,6 +21,15 @@ - src: "config.yaml.j2" dest: "{{ LITELLM_CONFIG_PATH_HOST }}" +- name: "๐Ÿงฎ Freeze the routing contract the CLI test asserts against" + ansible.builtin.set_fact: + LITELLM_TEST_CONTRACT: + ollama: "{{ LITELLM_OLLAMA_BACKEND | bool }}" + lmstudio: "{{ LITELLM_LMSTUDIO_BACKEND | bool }}" + served: "{{ LITELLM_CHAT_MODEL_SERVED | bool }}" + delegate_to: "{{ STACK_HOST }}" + run_once: true + - name: "๐Ÿ”ง Setup container network for {{ application_id }}" include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/network/routine.yml') }}" vars: diff --git a/roles/svc-ai-litellm/templates/test.env.j2 b/roles/svc-ai-litellm/templates/test.env.j2 index 42b48bb3c7..a3a33b2cc6 100644 --- a/roles/svc-ai-litellm/templates/test.env.j2 +++ b/roles/svc-ai-litellm/templates/test.env.j2 @@ -1,15 +1,15 @@ -{%- set _ollama_models = (lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') | map(attribute='alias') | list) if LITELLM_OLLAMA_BACKEND | bool else [] -%} +{%- set _ollama_models = (lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') | map(attribute='alias') | list) if LITELLM_TEST_CONTRACT.ollama | bool else [] -%} {%- set _lmstudio_declared = lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') | map(attribute='alias') | list -%} {%- set _lmstudio_only = _lmstudio_declared | reject('in', _ollama_models) | list -%} {%- set _remote = LITELLM_REMOTE_BACKENDS | selectattr('key') | map(attribute='alias') | list -%} -{%- set _expected = _ollama_models + (_lmstudio_only if LITELLM_LMSTUDIO_BACKEND | bool else []) + _remote -%} +{%- set _expected = _ollama_models + (_lmstudio_only if LITELLM_TEST_CONTRACT.lmstudio | bool else []) + _remote -%} LITELLM_CONTAINER={{ CLI_LOCAL_CID }} LITELLM_PORT={{ LITELLM_PORT }} LITELLM_MASTER_KEY_B64={{ LITELLM_MASTER_KEY | b64encode }} LITELLM_CHAT_MODEL={{ LITELLM_CHAT_MODEL }} -LITELLM_CHAT_MODEL_SERVED={{ LITELLM_CHAT_MODEL_SERVED | bool | string | lower }} -LITELLM_OLLAMA_ENABLED={{ LITELLM_OLLAMA_BACKEND | bool | string | lower }} -LITELLM_LMSTUDIO_ENABLED={{ LITELLM_LMSTUDIO_BACKEND | bool | string | lower }} +LITELLM_CHAT_MODEL_SERVED={{ LITELLM_TEST_CONTRACT.served | bool | string | lower }} +LITELLM_OLLAMA_ENABLED={{ LITELLM_TEST_CONTRACT.ollama | bool | string | lower }} +LITELLM_LMSTUDIO_ENABLED={{ LITELLM_TEST_CONTRACT.lmstudio | bool | string | lower }} LITELLM_LMSTUDIO_ALIASES='{{ _lmstudio_only | to_json }}' LITELLM_REMOTE_ALIASES='{{ _remote | to_json }}' LITELLM_EXPECTED_MODELS='{{ _expected | to_json }}' From e90cac4ee286aafce211e7f19f7a876763b2659a Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 02:25:53 +0200 Subject: [PATCH 027/197] test(lint): forbid no_log in the test roles roles/test-e2e-cli and roles/test-e2e-playwright run only in CI, against a throwaway cluster whose credentials are generated per deploy and discarded with it. no_log there protects nothing and costs the one thing a red test job exists for: a masked task reports that its output was hidden, and the whole matrix round has to run again to learn what a single assertion saw. The rule is absolute, so it ships without a nocheck escape hatch. A test task whose output looks too sensitive to print calls for extracting the field the assertion reads, not for blinding the task. A second case asserts the glob still reaches the roles. A path rule that matches nothing passes forever, which is how test_gateway_backend_registered nearly went quiet earlier today. Both roles are clean right now, so this starts green. Reproduced against an injected no_log in run_one.yml to confirm it is not vacuous. Note for whoever hits it: test_credential_tasks_are_masked scans roles/*/tasks and roles/*/handlers without excluding these two, and requires no_log on a task whose printed arguments name a password, secret, token, api_key or private_key. Nothing in the test roles trips it today, but a future task that does would deadlock the two rules. The exemption belongs in that lint's file selection. Co-Authored-By: Claude Opus 5 (1M context) --- .../roles/test_no_no_log_in_test_roles.py | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/lint/ansible/roles/test_no_no_log_in_test_roles.py diff --git a/tests/lint/ansible/roles/test_no_no_log_in_test_roles.py b/tests/lint/ansible/roles/test_no_no_log_in_test_roles.py new file mode 100644 index 0000000000..736e667c90 --- /dev/null +++ b/tests/lint/ansible/roles/test_no_no_log_in_test_roles.py @@ -0,0 +1,102 @@ +"""Lint: a test role must never carry ``no_log``. + +``roles/test-*`` run only in CI, against a throwaway cluster whose credentials +are generated per deploy and thrown away with it. ``no_log`` there protects +nothing and costs the one thing a red test job is for: the output that says +what failed. A masked task reports ``the output has been hidden`` and the +operator has to re-run the whole matrix round to learn what a single assertion +saw. + +The rule is absolute, so it carries no ``nocheck`` escape hatch. When a test +task's output looks too sensitive to print, extract the field the assertion +needs instead of blinding the task. +""" + +from __future__ import annotations + +import re +import unittest +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from utils.annotations.message import in_github_actions, warning +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +if TYPE_CHECKING: + from pathlib import Path + +_NO_LOG_RE = re.compile(r"^\s*no_log\s*:") + + +@dataclass(frozen=True) +class Finding: + path: str + line: int + text: str + + +def _collect_findings(root: Path) -> list[Finding]: + findings: list[Finding] = [] + for yaml_file in sorted(root.glob("roles/test-*/**/*.yml")): + try: + lines = read_text(str(yaml_file)).splitlines() + except OSError: + continue + rel = yaml_file.relative_to(root).as_posix() + findings.extend( + Finding(rel, number, line.strip()) + for number, line in enumerate(lines, start=1) + if _NO_LOG_RE.match(line) + ) + return findings + + +def _fix_hint(rel: str) -> str: + return ( + f"{rel} uses no_log in a test role. Test roles run in CI against " + "throwaway credentials, so no_log hides the failure output without " + "protecting anything. Drop it; if a value is genuinely too large or " + "noisy to print, extract the field the assertion reads." + ) + + +class TestNoNoLogInTestRoles(unittest.TestCase): + def test_test_roles_carry_no_no_log(self) -> None: + findings = _collect_findings(PROJECT_ROOT) + + for finding in findings: + warning( + _fix_hint(finding.path), + title="no_log in a test role", + file=finding.path, + line=finding.line, + ) + + if findings and not in_github_actions(): + print() + print(f"[WARNING] no_log in test roles ({len(findings)}):") + for finding in findings: + print(f"- {finding.path}:{finding.line}: {finding.text}") + + if findings: + self.fail( + f"{len(findings)} no_log entr(ies) in roles/test-*:\n" + + "\n".join( + f"{f.path}:{f.line}: {_fix_hint(f.path)}" for f in findings + ) + ) + + def test_the_scan_reaches_the_test_roles(self) -> None: + """A glob that matches nothing would pass vacuously forever.""" + scanned = sorted(PROJECT_ROOT.glob("roles/test-*/**/*.yml")) + self.assertTrue( + scanned, + "no YAML found under roles/test-*; the glob no longer reaches the " + "test roles, so the rule above cannot fire", + ) + + +if __name__ == "__main__": + unittest.main() From 15a6251d729e6cee74829261d403a4f74276be20 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 03:35:47 +0200 Subject: [PATCH 028/197] fix(api): resolve credentials against the declaration an inventory override hides API is a plain mapping, so an inventory that configures three providers replaces the whole declaration from group_vars/all/18_api.yml instead of merging into it. inventories/development/default.yml declares openai, anthropic and openrouter, so github, google and jira did not exist in CI at all and lookup('api', 'github.client_id') raised. A raising lookup inside a value the applications renderer is templating does not fail the render: _templar_render_best_effort returns the string untouched and, for a value carrying a non-env lookup(, refuses the fallback as well. The raw template text reached plugin_value.enabled | bool, which coerced it to False and emitted the deprecation warning that fails the deploy gate. Run 34893363518, job 104153194109 lost web-app-nextcloud to exactly that, for three integrations at once. The configvalue entries would have stored template text as the client id. utils/api.py merges the scope's override over the declaration, so an override narrows values and never the provider set, and holds the enable rule: every credential a provider declares must carry a value. The api lookup reads through it; the new api_enabled lookup gates on it and returns a real boolean instead of a string a filter has to coerce. The same class sat in group_vars/all/16_ai.yml and roles/svc-ai-litellm/meta/secrets.yml as bare API.. dereferences; both now go through the lookups. Two lints keep it shut. test_addon_enabled_renders_to_bool renders the whole applications map against a scope with API blanked, the most hostile override an inventory may legally write, and requires every addon gate to come out a boolean. test_api_accessed_via_lookup forbids a bare API dereference in Jinja under roles/, group_vars/ and inventories/. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/16_ai.yml | 6 +- group_vars/all/18_api.yml | 9 +- plugins/lookup/api.py | 14 +- plugins/lookup/api_enabled.py | 40 +++++ roles/svc-ai-litellm/meta/secrets.yml | 6 +- .../meta/addons/integration_github.yml | 2 +- .../meta/addons/integration_google.yml | 2 +- .../meta/addons/integration_jira.yml | 2 +- .../test_addon_enabled_renders_to_bool.py | 137 ++++++++++++++++++ .../variables/test_api_accessed_via_lookup.py | 105 ++++++++++++++ tests/unit/python/utils/test_api.py | 51 +++++++ utils/api.py | 108 ++++++++++++++ 12 files changed, 462 insertions(+), 20 deletions(-) create mode 100644 plugins/lookup/api_enabled.py create mode 100644 tests/lint/ansible/roles/test_addon_enabled_renders_to_bool.py create mode 100644 tests/lint/ansible/variables/test_api_accessed_via_lookup.py create mode 100644 tests/unit/python/utils/test_api.py create mode 100644 utils/api.py diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index 8191692603..65c6f27bcd 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -33,9 +33,9 @@ AI_LOCAL_MODELS: source: https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF AI_REMOTE_ALIASES: >- - {{ ['openai/gpt-4o-mini'] * ((API.openai.api_key | string | length > 0) | int) - + ['anthropic/claude-sonnet-4-5'] * ((API.anthropic.api_key | string | length > 0) | int) - + ['openrouter/auto'] * ((API.openrouter.api_key | string | length > 0) | int) }} + {{ ['openai/gpt-4o-mini'] * (lookup('api_enabled', 'openai') | int) + + ['anthropic/claude-sonnet-4-5'] * (lookup('api_enabled', 'anthropic') | int) + + ['openrouter/auto'] * (lookup('api_enabled', 'openrouter') | int) }} LITELLM_CHAT_MODEL: "{{ (lookup('config', 'svc-ai-ollama', 'services.ollama.preload_models') | map(attribute='alias') | reject('search', 'embed') | list | first | default('')) if 'svc-ai-ollama' in LITELLM_BACKEND_ROLES else ((lookup('config', 'svc-ai-lmstudio', 'services.lmstudio.preload_models') | map(attribute='alias') diff --git a/group_vars/all/18_api.yml b/group_vars/all/18_api.yml index 28ecf9d1fb..5dd167098a 100644 --- a/group_vars/all/18_api.yml +++ b/group_vars/all/18_api.yml @@ -1,10 +1,9 @@ --- # Central SPOT for proprietary / external (non-self-hosted) API credentials. -# Each entry is empty by default; an integration that depends on a provider is -# only activated when its credentials are present (see the addons that gate -# their `enabled` on `API.`). Override per provider in the (vault- -# encrypted) inventory; access ONLY via lookup('api', '.'). -# nocheck: unused-var: consumed by the api lookup plugin (plugins/lookup/api.py) via lookup('api', ...), not referenced directly in YAML/Jinja. +# Each entry is empty by default. Override per provider in the (vault- +# encrypted) inventory; read ONLY via lookup('api', '.') and +# gate ONLY on lookup('api_enabled', '') (see utils/api.py). +# nocheck: unused-var: consumed by the api lookup plugins (plugins/lookup/api.py, plugins/lookup/api_enabled.py) via lookup('api', ...), not referenced directly in YAML/Jinja. API: anthropic: api_key: "" diff --git a/plugins/lookup/api.py b/plugins/lookup/api.py index 9740e8d16b..a10e26c2f2 100644 --- a/plugins/lookup/api.py +++ b/plugins/lookup/api.py @@ -5,13 +5,16 @@ from typing import Any from ansible.errors import AnsibleError from ansible.plugins.lookup import LookupBase +from utils.api import resolve_api + class LookupModule(LookupBase): """ lookup('api', '.') e.g. lookup('api', 'github.client_id') Single point of truth for proprietary / external API credentials. Resolves a - dotted path against the global ``API`` mapping (group_vars/all/18_api.yml). + dotted path against the effective ``API`` mapping: the declaration in + group_vars/all/18_api.yml with the calling scope's override merged over it. Routing every access through this plugin keeps one place to change how proprietary API credentials are sourced (vault, env, secret manager, a different provider abstraction) without touching the call sites. @@ -31,11 +34,10 @@ class LookupModule(LookupBase): if not path: raise AnsibleError("lookup('api'): empty key path.") - api = variables.get("API", {}) - if templar is not None: - api = templar.template(api) - if not isinstance(api, dict): - raise AnsibleError("lookup('api'): global 'API' is not a mapping.") + try: + api = resolve_api(variables, templar=templar) + except TypeError as error: + raise AnsibleError(f"lookup('api'): {error}") from error node: Any = api for part in path.split("."): diff --git a/plugins/lookup/api_enabled.py b/plugins/lookup/api_enabled.py new file mode 100644 index 0000000000..c5f189070b --- /dev/null +++ b/plugins/lookup/api_enabled.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase + +from utils.api import provider_enabled, resolve_api + + +class LookupModule(LookupBase): + """ + lookup('api_enabled', '') e.g. lookup('api_enabled', 'github') + + True when every credential the provider declares in group_vars/all/18_api.yml + carries a value in the effective ``API`` mapping. Integrations that depend on + a proprietary provider gate on this instead of spelling the emptiness test + out per call site, so an added credential key extends the gate everywhere and + the result is a real boolean rather than a string a filter has to coerce. + """ + + def run(self, terms, variables: dict[str, Any] | None = None, **kwargs): + if not terms or len(terms) != 1: + raise AnsibleError( + "lookup('api_enabled') takes exactly one term: '' " + "(e.g. 'github')." + ) + + templar = getattr(self, "_templar", None) + variables = variables or getattr(templar, "available_variables", {}) or {} + + provider = str(terms[0]).strip() + if not provider: + raise AnsibleError("lookup('api_enabled'): empty provider name.") + + try: + api = resolve_api(variables, templar=templar) + return [provider_enabled(api, provider)] + except (KeyError, TypeError) as error: + raise AnsibleError(f"lookup('api_enabled'): {error}") from error diff --git a/roles/svc-ai-litellm/meta/secrets.yml b/roles/svc-ai-litellm/meta/secrets.yml index 47c983f31c..a2fb9ce606 100644 --- a/roles/svc-ai-litellm/meta/secrets.yml +++ b/roles/svc-ai-litellm/meta/secrets.yml @@ -15,14 +15,14 @@ credentials: OpenAI moved from sk- to sk-proj- without notice, and a schema that pinned the prefix would reject a valid key" type: "string" regex: "\\S{20,}" - default: "{{ API.openai.api_key }}" + default: "{{ lookup('api', 'openai.api_key') }}" anthropic_api_key: description: "Anthropic API key; empty leaves the anthropic/* models unpublished" type: "string" regex: "\\S{20,}" - default: "{{ API.anthropic.api_key }}" + default: "{{ lookup('api', 'anthropic.api_key') }}" openrouter_api_key: description: "OpenRouter API key; empty leaves openrouter/auto unpublished" type: "string" regex: "\\S{20,}" - default: "{{ API.openrouter.api_key }}" + default: "{{ lookup('api', 'openrouter.api_key') }}" diff --git a/roles/web-app-nextcloud/meta/addons/integration_github.yml b/roles/web-app-nextcloud/meta/addons/integration_github.yml index 5e3f59fab0..b409c441a0 100644 --- a/roles/web-app-nextcloud/meta/addons/integration_github.yml +++ b/roles/web-app-nextcloud/meta/addons/integration_github.yml @@ -1,5 +1,5 @@ --- -enabled: "{{ (lookup('api', 'github.client_id') | length > 0) and (lookup('api', 'github.client_secret') | length > 0) }}" +enabled: "{{ lookup('api_enabled', 'github') }}" required: true mechanism: plugin source: upstream diff --git a/roles/web-app-nextcloud/meta/addons/integration_google.yml b/roles/web-app-nextcloud/meta/addons/integration_google.yml index 98e6def4dd..c96349ab13 100644 --- a/roles/web-app-nextcloud/meta/addons/integration_google.yml +++ b/roles/web-app-nextcloud/meta/addons/integration_google.yml @@ -1,5 +1,5 @@ --- -enabled: "{{ (lookup('api', 'google.client_id') | length > 0) and (lookup('api', 'google.client_secret') | length > 0) }}" +enabled: "{{ lookup('api_enabled', 'google') }}" required: true mechanism: plugin source: upstream diff --git a/roles/web-app-nextcloud/meta/addons/integration_jira.yml b/roles/web-app-nextcloud/meta/addons/integration_jira.yml index 0b08bd9d25..94c1f2a096 100644 --- a/roles/web-app-nextcloud/meta/addons/integration_jira.yml +++ b/roles/web-app-nextcloud/meta/addons/integration_jira.yml @@ -1,5 +1,5 @@ --- -enabled: "{{ (lookup('api', 'jira.client_id') | length > 0) and (lookup('api', 'jira.client_secret') | length > 0) }}" +enabled: "{{ lookup('api_enabled', 'jira') }}" required: true mechanism: plugin source: upstream diff --git a/tests/lint/ansible/roles/test_addon_enabled_renders_to_bool.py b/tests/lint/ansible/roles/test_addon_enabled_renders_to_bool.py new file mode 100644 index 0000000000..b69de637d4 --- /dev/null +++ b/tests/lint/ansible/roles/test_addon_enabled_renders_to_bool.py @@ -0,0 +1,137 @@ +"""Every addon ``enabled`` must render to a boolean, even on a bare scope. + +Rationale +========= +``meta/addons/*.yml`` is rendered inside the applications SPOT and consumed as +``plugin_value.enabled | bool``. An expression that cannot resolve does not +fail: ``utils.templating.ansible._templar_render_best_effort`` returns the +string untouched, and for a value carrying a non-env ``lookup(`` it refuses the +fallback as well. The template text reaches the filter, which coerces it to +False and emits:: + + [DEPRECATION WARNING]: The `bool` filter coerced invalid value + "{{ ... }}" (str) to False. + +The deploy gate fails on that warning, and the addon is off in every run that +did not warn loudly enough to be noticed. Run 34893363518, job 104153194109 +lost web-app-nextcloud this way. + +The check renders the whole applications map against a scope carrying only what +every run has -- group_vars plus the deployed role names -- and blanks ``API`` +to the most hostile override an inventory may legally write. Anything that +survives as template text resolves only against something the render scope does +not carry. +""" + +from __future__ import annotations + +import unittest +from typing import Any + +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar + +from utils.cache.applications import ( + _MERGED_APPLICATIONS_CACHE, + get_merged_applications, +) +from utils.cache.yaml import load_yaml_any + +from . import PROJECT_ROOT + +_ADDONS_GLOB = "*/meta/addons/*.yml" +_BOOL_LITERALS = frozenset( + {"true", "false", "yes", "no", "on", "off", "1", "0", "y", "n"} +) + + +def _role_names() -> list[str]: + return sorted( + path.name for path in (PROJECT_ROOT / "roles").iterdir() if path.is_dir() + ) + + +def _hostile_scope() -> dict[str, Any]: + variables: dict[str, Any] = {} + for path in sorted((PROJECT_ROOT / "group_vars" / "all").glob("*.yml")): + data = load_yaml_any(path) or {} + if isinstance(data, dict): + variables.update(data) + variables["group_names"] = _role_names() + variables["API"] = {} + return variables + + +def _rendered_addons() -> dict[str, dict[str, Any]]: + variables = _hostile_scope() + templar = Templar(loader=DataLoader(), variables=dict(variables)) + templar.available_variables = dict(variables) + + _MERGED_APPLICATIONS_CACHE.clear() + try: + applications = get_merged_applications( + variables=dict(variables), templar=templar + ) + finally: + _MERGED_APPLICATIONS_CACHE.clear() + + return { + app_id: config["addons"] + for app_id, config in applications.items() + if isinstance(config, dict) and isinstance(config.get("addons"), dict) + } + + +def _verdict(value: Any) -> str: + if isinstance(value, bool): + return "" + if not isinstance(value, str): + return f"is {type(value).__name__}, not a boolean" + if "{{" in value or "{%" in value: + return f"stayed template text: {value}" + if value.strip().lower() not in _BOOL_LITERALS: + return f"is not a boolean literal: {value!r}" + return "" + + +class TestAddonEnabledRendersToBool(unittest.TestCase): + def test_every_addon_enabled_resolves(self) -> None: + findings = [ + f" - roles/{app_id}/meta/addons/{addon_id}.yml: 'enabled' {verdict}" + for app_id, addons in _rendered_addons().items() + for addon_id, spec in sorted(addons.items()) + if isinstance(spec, dict) + for verdict in [_verdict(spec.get("enabled"))] + if verdict + ] + + if findings: + self.fail( + f"{len(findings)} addon gate(s) do not render to a boolean on a " + "scope without inventory extras, so '| bool' coerces the leftover " + "text to False and the deploy fails on the deprecation warning:\n" + + "\n".join(findings) + + "\n\nResolve the value through a lookup that carries its own " + "source instead of a variable the render scope may not hold: " + "lookup('api_enabled', '') gates on credentials, " + "lookup('config', ...) on another application's settings." + ) + + def test_the_scan_reaches_templated_gates(self) -> None: + """A scope that resolved nothing, or addons without gates, passes for free.""" + templated = [ + path + for path in sorted((PROJECT_ROOT / "roles").glob(_ADDONS_GLOB)) + if "{{" in str((load_yaml_any(path) or {}).get("enabled", "")) + ] + self.assertTrue( + templated, + "no addon declares a templated 'enabled', so the rule above is vacuous", + ) + + rendered = _rendered_addons() + self.assertTrue(rendered, "no application carries addons; nothing was checked") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/lint/ansible/variables/test_api_accessed_via_lookup.py b/tests/lint/ansible/variables/test_api_accessed_via_lookup.py new file mode 100644 index 0000000000..803ab4177d --- /dev/null +++ b/tests/lint/ansible/variables/test_api_accessed_via_lookup.py @@ -0,0 +1,105 @@ +"""``API`` is reachable through its lookups only, never as a bare variable. + +Rationale +========= +A bare ``API.github.client_id`` resolves against whatever the scope happens to +hold, and an inventory override replaces the declaration rather than merging +into it -- ``utils/api.py`` carries the mechanism and the consequences. +``lookup('api', '.')`` and ``lookup('api_enabled', '')`` +merge the override over ``group_vars/all/18_api.yml``, so an override narrows +values and never the provider set. Routing every access through them also keeps +one place to change how credentials are sourced. + +Scope +===== +Jinja in ``roles/``, ``group_vars/`` and ``inventories/``. An ``API:`` key +being *declared* is not an access; only a dereference inside ``{{ }}`` or +``{% %}`` is. Test fixtures legitimately build the bare form to exercise the +renderer, so ``tests/`` is out of scope. +""" + +from __future__ import annotations + +import re +import unittest +from dataclasses import dataclass +from pathlib import Path + +from utils.cache.files import iter_project_files_with_content + +from . import PROJECT_ROOT + +_SCOPES = ("roles", "group_vars", "inventories") +_SUFFIXES = (".yml", ".yaml", ".j2") +_JINJA_BLOCK = re.compile(r"\{\{(.*?)\}\}|\{%(.*?)%\}", re.DOTALL) +_BARE_API = re.compile(r"(? list[tuple[str, str]]: + prefixes = tuple(f"{scope}/" for scope in _SCOPES) + scoped: list[tuple[str, str]] = [] + for path, text in iter_project_files_with_content( + extensions=_SUFFIXES, exclude_tests=True + ): + rel = Path(path).relative_to(PROJECT_ROOT).as_posix() + if rel.startswith(prefixes): + scoped.append((rel, text)) + return sorted(scoped) + + +def _findings() -> list[Finding]: + findings: list[Finding] = [] + for rel, text in _scoped_files(): + for match in _JINJA_BLOCK.finditer(text): + expression = match.group(1) or match.group(2) or "" + if not _BARE_API.search(expression): + continue + line = text.count("\n", 0, match.start()) + 1 + findings.append(Finding(rel, line, " ".join(expression.split()))) + return findings + + +class TestApiAccessedViaLookup(unittest.TestCase): + def test_no_jinja_dereferences_api_directly(self) -> None: + findings = _findings() + + if findings: + self.fail( + f"{len(findings)} expression(s) read 'API' as a bare variable, which " + "breaks against an inventory that overrides only some providers:\n" + + "\n".join( + f" - {f.path}:{f.line}: {{{{ {f.expression} }}}}" for f in findings + ) + + "\n\nRead the value with lookup('api', '.') and gate " + "on lookup('api_enabled', '')." + ) + + def test_the_scan_reaches_jinja_in_every_scope(self) -> None: + """A suffix or scope typo would make the rule above pass forever.""" + scanned = _scoped_files() + for scope in _SCOPES: + with self.subTest(scope=scope): + blocks = sum( + len(_JINJA_BLOCK.findall(text)) + for rel, text in scanned + if rel.startswith(f"{scope}/") + ) + self.assertTrue(blocks, f"no Jinja found under {scope}/") + + def test_the_pattern_separates_a_dereference_from_a_declaration(self) -> None: + self.assertTrue(_BARE_API.search("API.github.client_id | length")) + self.assertTrue(_BARE_API.search("API['github']")) + self.assertFalse(_BARE_API.search("lookup('api', 'github.client_id')")) + self.assertFalse(_BARE_API.search("LITELLM_API.url")) + self.assertFalse(_BARE_API.search("item.API.key")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/test_api.py b/tests/unit/python/utils/test_api.py new file mode 100644 index 0000000000..1a48d495b9 --- /dev/null +++ b/tests/unit/python/utils/test_api.py @@ -0,0 +1,51 @@ +import unittest + +from utils.api import declared_api, provider_enabled, resolve_api + + +class TestResolveApi(unittest.TestCase): + def test_a_partial_override_keeps_the_declared_providers(self) -> None: + declared = declared_api() + self.assertIn("github", declared) + + resolved = resolve_api({"API": {"openai": {"api_key": "sk-test"}}}) + + self.assertEqual(set(resolved), set(declared)) + self.assertEqual(resolved["openai"]["api_key"], "sk-test") + self.assertEqual(resolved["github"]["client_id"], "") + + def test_an_override_wins_per_key(self) -> None: + resolved = resolve_api({"API": {"github": {"client_id": "ID42"}}}) + + self.assertEqual(resolved["github"]["client_id"], "ID42") + self.assertEqual(resolved["github"]["client_secret"], "") + + def test_a_missing_api_falls_back_to_the_declaration(self) -> None: + self.assertEqual(resolve_api({}), declared_api()) + + def test_a_non_mapping_api_is_an_error(self) -> None: + with self.assertRaises(TypeError): + resolve_api({"API": "nope"}) + + +class TestProviderEnabled(unittest.TestCase): + def test_every_credential_must_carry_a_value(self) -> None: + api = {"github": {"client_id": "ID42", "client_secret": "SEC42"}} + + self.assertTrue(provider_enabled(api, "github")) + + def test_one_empty_credential_disables_the_provider(self) -> None: + api = {"github": {"client_id": "ID42", "client_secret": " "}} + + self.assertFalse(provider_enabled(api, "github")) + + def test_a_provider_without_credentials_is_disabled(self) -> None: + self.assertFalse(provider_enabled({"github": {}}, "github")) + + def test_an_undeclared_provider_is_an_error(self) -> None: + with self.assertRaises(KeyError): + provider_enabled({}, "github") + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/api.py b/utils/api.py new file mode 100644 index 0000000000..792ec75be4 --- /dev/null +++ b/utils/api.py @@ -0,0 +1,108 @@ +"""Proprietary API credential SPOT, read from group_vars/all/18_api.yml. + +``API`` is a plain mapping, so an inventory that configures three providers +replaces the whole declaration instead of merging into it -- see +``inventories/development/default.yml``, which declares openai, anthropic and +openrouter and therefore hides github, google and jira. Resolving a hidden +provider then raises, and a raising lookup inside a value the applications +renderer is templating leaves the raw ``{{ ... }}`` text in place: the consumer +stores template text, and ``| bool`` coerces it to False with a deprecation +warning that fails the deploy on the warning gate. + +Merging the scope's override over the declared defaults keeps every declared +provider resolvable, so an override narrows values, never the provider set. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import yaml + +from utils import PROJECT_ROOT +from utils.cache.base import _deep_merge +from utils.cache.files import read_text + +_API_SPOT_FILE = str(PROJECT_ROOT / "group_vars" / "all" / "18_api.yml") + + +def declared_api() -> dict[str, Any]: + """The ``API`` mapping as declared in the group_vars SPOT. + + Returns: + Provider name to credential mapping, every value a plain string. + + Raises: + TypeError: the SPOT does not declare ``API`` as a mapping. + """ + document = yaml.safe_load(read_text(_API_SPOT_FILE)) or {} + declared = document.get("API") + if not isinstance(declared, Mapping): + raise TypeError(f"{_API_SPOT_FILE} does not declare 'API' as a mapping.") + return dict(declared) + + +def resolve_api( + variables: Mapping[str, Any] | None, templar: Any = None +) -> dict[str, Any]: + """The effective ``API`` mapping for a render scope. + + Args: + variables: the scope's variables; its ``API`` entry overrides the SPOT. + templar: optional templar; renders the override and the merged values, + so an inventory may source a credential from an env lookup. + + Returns: + The SPOT declaration with the scope's override merged over it. + + Raises: + TypeError: the scope's ``API`` is present but not a mapping. + """ + declared = declared_api() + override: Any = (variables or {}).get("API") + if override is not None and not isinstance(override, Mapping) and templar: + override = _templated(templar, override) + if override is None: + return declared + if not isinstance(override, Mapping): + raise TypeError("'API' is defined but is not a mapping.") + merged = _deep_merge(declared, dict(override)) + if templar is None: + return merged + rendered = _templated(templar, merged) + return dict(rendered) if isinstance(rendered, Mapping) else merged + + +def _templated(templar: Any, value: Any) -> Any: + from ansible.errors import AnsibleError + + try: + return templar.template(value, fail_on_undefined=False) + except (AnsibleError, TypeError, ValueError): + return value + + +def provider_enabled(api: Mapping[str, Any], provider: str) -> bool: + """Whether every credential of a provider carries a value. + + Args: + api: an effective ``API`` mapping, as returned by ``resolve_api``. + provider: the provider name, e.g. ``github``. + + Returns: + True when the provider declares at least one credential and none of + them is empty. + + Raises: + KeyError: the provider is not declared in the SPOT. + """ + credentials = api.get(provider) + if not isinstance(credentials, Mapping): + raise KeyError( + f"unknown API provider {provider!r}; declare it under 'API' in " + f"{_API_SPOT_FILE}." + ) + if not credentials: + return False + return all(str(value).strip() for value in credentials.values()) From de3f93228c07dc89cbc5a731c06bdf0a4eacb4dd Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 03:36:41 +0200 Subject: [PATCH 029/197] docs(tor): regenerate the role table for the onion service lifecycle 691911279f moved roles/svc-net-tor/meta/services.yml from alpha to beta but staged only the source. The README role table is generated from those declarations, so the two had drifted apart. No behaviour attaches to the table; cli/meta/ci/validate.py reads the meta file. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 306cd45a83..9963b1ec95 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ Every invokable role in the tested lifecycle envelope, with its upstream homepag | [SuiteCRM](roles/web-app-suitecrm/) | beta | Open-source CRM platform for sales, service, cases, and campaign management with workflow automation | [๐ŸŒ](https://suitecrm.com/) [๐ŸŽฌ](https://www.youtube.com/watch?v=pOAs_C3Fp_E) [๐Ÿ› ๏ธ](roles/web-app-suitecrm/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Swapfile](roles/svc-opt-swapfile/) | beta | On-target automation for creating and activating swap space via a cloned swapfile script | [๐Ÿ› ๏ธ](roles/svc-opt-swapfile/README.md#quick-setup) | | | [Taiga](roles/web-app-taiga/) | beta | Open-source agile project management platform for Scrum, Kanban, and hybrid team workflows | [๐ŸŒ](https://taiga.io/) [๐ŸŽฌ](https://www.youtube.com/watch?v=TYPX8nQEU5I) [๐Ÿ› ๏ธ](roles/web-app-taiga/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-app-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | -| [Tor Onion Service](roles/svc-net-tor/) | alpha | Tor daemon publishing the node as a v3 onion service | [๐Ÿ› ๏ธ](roles/svc-net-tor/README.md#quick-setup) | [Asset Mirror (Privacy Proxy)](roles/web-svc-mirror/) | +| [Tor Onion Service](roles/svc-net-tor/) | beta | Tor daemon publishing the node as a v3 onion service | [๐Ÿ› ๏ธ](roles/svc-net-tor/README.md#quick-setup) | [Asset Mirror (Privacy Proxy)](roles/web-svc-mirror/) | | [Typesense](roles/svc-db-typesense/) | beta | Central Dockerized search engine shared by multiple apps with per-consumer scoped API keys | [๐ŸŒ](https://typesense.org/) [๐Ÿ› ๏ธ](roles/svc-db-typesense/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Tor Onion Service](roles/svc-net-tor/) | | [Unbound](roles/svc-dns-unbound/) | beta | Central recursive/forwarding DNSSEC resolver on a static IP for consumer stacks to target as their DNS | [๐ŸŒ](https://nlnetlabs.nl/projects/unbound/about/) [๐Ÿ› ๏ธ](roles/svc-dns-unbound/README.md#quick-setup) | | | [Universal Logout](roles/web-svc-logout/) | beta | Central logout proxy for single-action sign-out across all OIDC-connected apps, with Nginx snippets and a live status UI | [๐Ÿ› ๏ธ](roles/web-svc-logout/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | From 3d3defcaaeb18f37180e5d8f4b5c6fb7eae11bb1 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 03:37:21 +0200 Subject: [PATCH 030/197] style(lint): wrap the no_log failure message the way autoformat does e90cac4ee2 staged the lint but not what the gate rewrote after it. Whitespace only; the join, the fix hint and the assertion are unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- tests/lint/ansible/roles/test_no_no_log_in_test_roles.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/lint/ansible/roles/test_no_no_log_in_test_roles.py b/tests/lint/ansible/roles/test_no_no_log_in_test_roles.py index 736e667c90..a4528978cf 100644 --- a/tests/lint/ansible/roles/test_no_no_log_in_test_roles.py +++ b/tests/lint/ansible/roles/test_no_no_log_in_test_roles.py @@ -83,9 +83,7 @@ class TestNoNoLogInTestRoles(unittest.TestCase): if findings: self.fail( f"{len(findings)} no_log entr(ies) in roles/test-*:\n" - + "\n".join( - f"{f.path}:{f.line}: {_fix_hint(f.path)}" for f in findings - ) + + "\n".join(f"{f.path}:{f.line}: {_fix_hint(f.path)}" for f in findings) ) def test_the_scan_reaches_the_test_roles(self) -> None: From 182ad13717f11bb3fe154f6c5b4240ef1b73bbd6 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 03:50:02 +0200 Subject: [PATCH 031/197] fix(api): let the SPOT merge survive a templar without the kwarg 15a6251d7 introduced the merge but tripped over three of the repository's own rules and over the lookup's existing contract test. _templated called templar.template(value, fail_on_undefined=False). A templar whose signature is template(self, value) raises TypeError on that kwarg, the except swallowed it, and the value came back unrendered: the lookup returned ['{{ vault_cf }}'] where ['secret-cf'] was due. It now falls back to the positional call, the way utils/cache/base.py has always done it. The SPOT read goes through load_yaml_any instead of yaml.safe_load, which tests/lint/repository/yaml/test_no_direct_calls.py requires so the file is parsed once per process. The module docstring drops the inventories/development/default.yml literal, which tests/unit/python/cli/administration/deploy/development/test_inventory_vars_file_spot.py whitelists by path. test_missing_api_global_raises_unknown_key asserted exactly the behaviour that broke CI: a scope without API had to raise. It now asserts the fallback, and two new cases pin the edges. A provider the scope omits still resolves; a provider the declaration does not know still raises. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/python/plugins/lookup/test_api.py | 14 ++++++++-- utils/api.py | 28 +++++++++++--------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/tests/unit/python/plugins/lookup/test_api.py b/tests/unit/python/plugins/lookup/test_api.py index eaa1ecedbb..f9dc81bb27 100644 --- a/tests/unit/python/plugins/lookup/test_api.py +++ b/tests/unit/python/plugins/lookup/test_api.py @@ -90,9 +90,19 @@ class TestApiLookup(unittest.TestCase): self.lookup.run(["cloudflare.api_token"], variables={"API": "nope"}) self.assertIn("mapping", str(ctx.exception)) - def test_missing_api_global_raises_unknown_key(self): + def test_missing_api_global_falls_back_to_the_declaration(self): + self.assertEqual(self.lookup.run(["cloudflare.api_token"], variables={})[0], "") + + def test_a_provider_the_scope_omits_still_resolves(self): + out = self.lookup.run( + ["github.client_id"], + variables={"API": {"cloudflare": {"api_token": "cf"}}}, + ) + self.assertEqual(out, [""]) + + def test_an_undeclared_provider_still_raises(self): with self.assertRaises(AnsibleError): - self.lookup.run(["cloudflare.api_token"], variables={}) + self.lookup.run(["nope.api_key"], variables=self.vars) def test_templar_resolves_jinja_values(self): self.lookup._templar = _FakeTemplar(mapping={"{{ vault_cf }}": "secret-cf"}) diff --git a/utils/api.py b/utils/api.py index 792ec75be4..b7c0f95891 100644 --- a/utils/api.py +++ b/utils/api.py @@ -1,13 +1,12 @@ """Proprietary API credential SPOT, read from group_vars/all/18_api.yml. ``API`` is a plain mapping, so an inventory that configures three providers -replaces the whole declaration instead of merging into it -- see -``inventories/development/default.yml``, which declares openai, anthropic and -openrouter and therefore hides github, google and jira. Resolving a hidden -provider then raises, and a raising lookup inside a value the applications -renderer is templating leaves the raw ``{{ ... }}`` text in place: the consumer -stores template text, and ``| bool`` coerces it to False with a deprecation -warning that fails the deploy on the warning gate. +replaces the whole declaration instead of merging into it, and every provider it +does not name disappears. Resolving a hidden provider then raises, and a raising +lookup inside a value the applications renderer is templating leaves the raw +``{{ ... }}`` text in place: the consumer stores template text, and ``| bool`` +coerces it to False with a deprecation warning that fails the deploy on the +warning gate. Merging the scope's override over the declared defaults keeps every declared provider resolvable, so an override narrows values, never the provider set. @@ -18,11 +17,9 @@ from __future__ import annotations from collections.abc import Mapping from typing import Any -import yaml - from utils import PROJECT_ROOT from utils.cache.base import _deep_merge -from utils.cache.files import read_text +from utils.cache.yaml import load_yaml_any _API_SPOT_FILE = str(PROJECT_ROOT / "group_vars" / "all" / "18_api.yml") @@ -36,8 +33,8 @@ def declared_api() -> dict[str, Any]: Raises: TypeError: the SPOT does not declare ``API`` as a mapping. """ - document = yaml.safe_load(read_text(_API_SPOT_FILE)) or {} - declared = document.get("API") + document = load_yaml_any(_API_SPOT_FILE, default_if_missing={}) or {} + declared = document.get("API") if isinstance(document, Mapping) else None if not isinstance(declared, Mapping): raise TypeError(f"{_API_SPOT_FILE} does not declare 'API' as a mapping.") return dict(declared) @@ -79,6 +76,13 @@ def _templated(templar: Any, value: Any) -> Any: try: return templar.template(value, fail_on_undefined=False) + except TypeError: + pass + except (AnsibleError, ValueError): + return value + + try: + return templar.template(value) except (AnsibleError, TypeError, ValueError): return value From 280e338cdc30e44e1b6005f2a4f2efde7ed4f037 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 04:05:07 +0200 Subject: [PATCH 032/197] fix(cache): retry the cert generator's image pull Run 34893363518 lost the fedora workspace job at certs.sh with exit 125. docker run resolved alpine:latest itself, the token request to auth.docker.io came back as 'read: connection reset by peer', and that single transport fault ended the job. Five of the six workspace jobs pulled the same image successfully in the same run, so the cause is transport, not a Docker Hub policy response, whose signature would be toomanyrequests after a successful token exchange. Every other network operation on this path already retries: scripts/github/runner/pull_with_retry.sh serves two workflows, scripts/image/build.sh has its own ladder for the build frontend, and cli/administration/deploy/development/compose.py retries compose up six times. This call site had no attempts at all, so pre-pull through the existing helper rather than a new loop. The attempt budget is 3x10s instead of the helper's 7x20s default because this runs as a prerequisite on the interactive local path, where two minutes of sleeping without a network would be felt. The call stays non-fatal so a locally cached image still carries an unreachable registry, the same semantics 0df75d83a6 established for the buildx bootstrap; set -euo pipefail is why it needs the if-guard. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/docker/cache/package-frontend/certs.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/docker/cache/package-frontend/certs.sh b/scripts/docker/cache/package-frontend/certs.sh index 22af789401..2e8f338f00 100755 --- a/scripts/docker/cache/package-frontend/certs.sh +++ b/scripts/docker/cache/package-frontend/certs.sh @@ -19,6 +19,11 @@ fi : "${INFINITO_CACHE_PACKAGE_FRONTEND_INIT_IMAGE:?Source scripts/meta/env/load.sh first}" ALPINE_IMAGE="${INFINITO_CACHE_PACKAGE_FRONTEND_INIT_IMAGE}" +if ! MAX_ATTEMPTS=3 RETRY_DELAY_SECONDS=10 \ + "${REPO_ROOT}/scripts/github/runner/pull_with_retry.sh" "${ALPINE_IMAGE}"; then + echo "[package-frontend-certs] could not pull ${ALPINE_IMAGE}; falling back to a cached copy" >&2 +fi + exec docker run --rm \ -v "${INFINITO_CACHE_PACKAGE_FRONTEND_CA_DIR}:/ca" \ -v "${INFINITO_CACHE_PACKAGE_FRONTEND_CERTS_DIR}:/certs" \ From af6d4a42cb33505bb0a907e07bc7cd3953b6e6ef Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 04:26:28 +0200 Subject: [PATCH 033/197] docs(lmstudio): describe the preload contract the role actually has f8da0fd122 made the pre-pull download model.source instead of the bare model.name, but the Features list still said an entry 'carries the LM Studio name to download'. An author following that sentence writes the search term that caused the failure: lms get answers 'Multiple models found. Automatically selecting the first one due to --yes' and fetches a different model, after which the gateway addresses a name nothing serves. Name all three fields with their roles. source is the repository that gets downloaded, name is the key LM Studio indexes it under and templates/config.yaml.j2 addresses as openai/, alias is what the gateway publishes. A second entry records that the pre-pull does not load. That is what makes the upstream 'No models loaded' body readable, and it is why svc-ai-litellm warms the gateway at deploy time rather than leaving the cold load to the first consumer. Measured on the image CI runs (sha256:48abf3b7e061): the URL form resolves the pinned repository and indexes it as qwen2.5-0.5b-instruct, the search-term form resolves to the Coder variant and indexes it as qwen2.5-coder-0.5b-instruct, and a chat completion against an empty lms ps loads the model just in time and answers 200. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-lmstudio/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/svc-ai-lmstudio/README.md b/roles/svc-ai-lmstudio/README.md index d313883f0a..bb671103ee 100644 --- a/roles/svc-ai-lmstudio/README.md +++ b/roles/svc-ai-lmstudio/README.md @@ -35,7 +35,8 @@ Solid `1:1` edges are fixed relationships; dashed `0..1` edges are conditional ( - **OpenAI-compatible endpoint:** The headless server answers `/v1` requests on port 1234 inside the container network. - **CPU inference:** The role pins the CPU build of the upstream image and passes no GPU device into the container. - **Persistent model store:** A named volume mounted at `/root/.lmstudio` keeps downloaded models and server settings across redeploys. -- **Preloaded models:** Every entry of `services.lmstudio.preload_models` is downloaded on the hosting node with `lms get`, overlapped and reaped like the Ollama pre-pull. Each entry carries the LM Studio `name` to download and the `alias` the gateway publishes it under. +- **Preloaded models:** Every entry of `services.lmstudio.preload_models` is downloaded on the hosting node with `lms get --gguf`, overlapped and reaped like the Ollama pre-pull. Each entry carries the `source` repository to download, the `name` LM Studio indexes that repository under and the gateway addresses it by, and the `alias` the gateway publishes it as. `source` is a full Hugging Face URL, never a search term: a search term resolves to whichever model the catalogue lists first. +- **Loading:** The pre-pull downloads without loading. LM Studio serves with just-in-time loading, so the first `/v1/chat/completions` naming a `name` loads that model; the gateway's warm task issues that first request at deploy time. - **Gateway backend:** The LiteLLM Gateway publishes those aliases, which are the names Ollama serves too, so the same model name routes to whichever local backend a deployment runs. - **Bounded resources:** The container is capped at 4 CPUs, 8 GB of memory and 2048 processes, and the role declares a minimum of 588 MB free storage for the image; downloaded models grow the volume beyond that. - **Backup integration:** Backup Docker Volumes snapshots the model volume when that role is present, without stopping the container. From 5b1ac73dd9edaf8fd8ce80f10b4ebe6434bab537 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 11:20:40 +0200 Subject: [PATCH 034/197] fix(domains): render the tor flags before the onion map reads them services.tor.enabled is Jinja in 99 of the 105 roles that declare it. A lookup('tls') or lookup('domains') inside a meta/*.yml value runs while the applications render is in flight, the re-entry guard hands it the unrendered tree, and _as_bool read the flag's own source text as False. The onion injection then did nothing for any app and the clearnet domain was returned. fa3797540a already described this and quarantined it: the render state joined the domains cache key so the poisoned map answers no later read. The value stayed wrong, which is enough until a call site bakes it into applications. roles/web-svc-asset/meta/services.yml:15 is the only one in the repo that does, and in run 34922994823 it froze https://file.infinito.test/assets into the payload while every vhost of that round was published under the node onion. web-app-dashboard then waited 60 times on a host openresty answers with ssl_reject_handshake, and the deploy died on TLSV1_UNRECOGNIZED_NAME. _resolved_flag renders enabled, exclusive and primary and raises when template text survives, the shape plugins/lookup/proxy_pass.py already uses. plugins/lookup/config.py and plugins/lookup/email.py render before coercing for the same reason; this module was the one that did not. Measured across the whole map: 44 of 12819 leaves change, none added, none removed, 17 roles. 42 are clearnet to onion. The two others are repairs as well: keycloak's csp frame-ancestors moves from https to http, which is what an onion node serves, and the nextcloud sociallogin provider name stops being a hostname and becomes the bounded name that test_sociallogin_provider_name.py documents. The new tests drive the same function with Jinja instead of the literal booleans the existing ones pass, which is why those never saw this. Three of the four fail without the fix. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/python/utils/cache/test_domains.py | 66 ++++++++++++++ utils/cache/domains.py | 91 +++++++++++++++++-- 2 files changed, 150 insertions(+), 7 deletions(-) diff --git a/tests/unit/python/utils/cache/test_domains.py b/tests/unit/python/utils/cache/test_domains.py index ae7b2b7c35..c884e16db2 100644 --- a/tests/unit/python/utils/cache/test_domains.py +++ b/tests/unit/python/utils/cache/test_domains.py @@ -337,5 +337,71 @@ class TestOnionDomainInjection(unittest.TestCase): self.assertEqual(out["web-app-x"], ["infinito.test", self.ONION]) +class TestOnionFlagsAreRendered(unittest.TestCase): + """`services.tor.*` reaches this map as Jinja in 99 of the 105 roles that + declare it, and as its own source text whenever the applications render is + still in flight. Reading that text as a boolean answers False and drops the + onion domains for every app, which is what froze a clearnet asset URL into + the applications payload of run 34922994823. + """ + + ONION = TestOnionDomainInjection.ONION + JINJA_ON = "{{ 'svc-net-tor' in group_names }}" + + _apps = TestOnionDomainInjection._apps + + def _templar(self, **variables): + from ansible.parsing.dataloader import DataLoader + from ansible.template import Templar + + templar = Templar(loader=DataLoader(), variables=dict(variables)) + templar.available_variables = dict(variables) + return templar + + def _inject(self, apps, **variables): + return cache_domains._inject_onion_domains( + {"web-app-x": ["x.infinito.test"]}, + apps, + "infinito.test", + self.ONION, + templar=self._templar(**variables), + variables=dict(variables), + ) + + def test_a_templated_enabled_flag_still_injects_the_onion(self): + out = self._inject( + self._apps(enabled=self.JINJA_ON), + group_names=["svc-net-tor", "web-app-x"], + ) + + self.assertEqual(out["web-app-x"], ["x.infinito.test", f"x.{self.ONION}"]) + + def test_a_templated_flag_resolving_false_leaves_the_clearnet_domain(self): + out = self._inject(self._apps(enabled=self.JINJA_ON), group_names=["web-app-x"]) + + self.assertEqual(out["web-app-x"], ["x.infinito.test"]) + + def test_templated_exclusive_replaces_the_clearnet_domain(self): + out = self._inject( + self._apps(enabled=self.JINJA_ON, exclusive=self.JINJA_ON), + group_names=["svc-net-tor", "web-app-x"], + ) + + self.assertEqual(out["web-app-x"], [f"x.{self.ONION}"]) + + def test_a_flag_that_cannot_be_rendered_is_an_error(self): + """Without a templar the text cannot resolve, and answering False + would drop the onion domains without saying so.""" + with self.assertRaisesRegex(ValueError, "services.tor.enabled"): + cache_domains._inject_onion_domains( + {"web-app-x": ["x.infinito.test"]}, + self._apps(enabled=self.JINJA_ON), + "infinito.test", + self.ONION, + templar=None, + variables={"group_names": ["svc-net-tor"]}, + ) + + if __name__ == "__main__": unittest.main() diff --git a/utils/cache/domains.py b/utils/cache/domains.py index 93defc8919..ee4bd569a4 100644 --- a/utils/cache/domains.py +++ b/utils/cache/domains.py @@ -32,6 +32,54 @@ def _as_bool(value: Any) -> bool: return bool(value) +def _resolved_flag( + value: Any, + *, + app: str, + name: str, + templar: Any, + variables: Any, + apps: Any, +) -> bool: + """A tor service flag as a boolean, never as the text of its own template. + + ``services.tor.*`` is declared as Jinja in 99 of the 105 roles that carry + it. When this map is built from an applications tree that is still being + rendered (``utils.cache.applications`` returns the raw tree while its own + render is in flight) the flag arrives as its source text, which every + boolean coercion reads as False. The onion injection would then silently + not happen and the clearnet domain would be frozen into whatever value the + render was producing. + + Args: + value: the declared flag, rendered or not. + app: application id, for the error message. + name: flag name under ``services.tor``, for the error message. + templar: templar to render with; ``None`` renders nothing. + variables: scope the template resolves against. + apps: applications tree seeded into the nested render. + + Returns: + The flag as a boolean. + + Raises: + ValueError: the flag still carries template text after rendering. + """ + if isinstance(value, str) and ("{{" in value or "{%" in value): + from utils.cache.base import _render_with_templar + + value = _render_with_templar( + value, templar=templar, variables=variables, raw_applications=apps + ) + if isinstance(value, str) and ("{{" in value or "{%" in value): + raise ValueError( + f"get_merged_domains: '{app}' services.tor.{name} stayed template " + f"text ({value!r}). Reading it as a boolean would drop the onion " + "domains without saying so." + ) + return _as_bool(value) + + def _onion_of(domain: str, primary: str, node_onion: str) -> str | None: """Return ``domain`` with the clearnet ``primary`` suffix swapped for the node onion, or None when the domain is not under ``primary``.""" @@ -48,6 +96,9 @@ def _inject_onion_domains( apps: dict[str, Any], primary: str, node_onion: str, + *, + templar: Any = None, + variables: Any = None, ) -> dict[str, Any]: """Add ``.`` domains for apps opting into onion routing. @@ -63,21 +114,38 @@ def _inject_onion_domains( ) registry = build_service_registry_from_applications(apps) + + def _flag(value: Any, app: str, name: str) -> bool: + return _resolved_flag( + value, + app=app, + name=name, + templar=templar, + variables=variables, + apps=apps, + ) + out: dict[str, Any] = {} for app, domains in merged.items(): tor = ((apps.get(app) or {}).get("services") or {}).get("tor") or {} - if not isinstance(domains, (list, dict)) or not _as_bool(tor.get("enabled")): + if not isinstance(domains, (list, dict)) or not _flag( + tor.get("enabled"), app, "enabled" + ): out[app] = domains continue - exclusive = _as_bool( + exclusive = _flag( resolve_service_config( apps, app, "tor", "exclusive", default=False, service_registry=registry - ) + ), + app, + "exclusive", ) - is_primary = _as_bool( + is_primary = _flag( resolve_service_config( apps, app, "tor", "primary", default=False, service_registry=registry - ) + ), + app, + "primary", ) if isinstance(domains, dict): out[app] = _inject_onion_into_dict( @@ -141,7 +209,9 @@ def get_merged_domains( issued while the applications render is in progress sees the unrendered tree, whose service flags are still Jinja strings; its map is cached under its own key so the render's own call sites reuse it without it ever being - served as the finished map. + served as the finished map. The tor flags are rendered before they are + read (``_resolved_flag``), so that mid-render map carries the same onion + domains the finished one does. """ from plugins.filter.canonical_domains_map import ( FilterModule as _CanonicalDomainsFilter, @@ -200,7 +270,14 @@ def get_merged_domains( node_onion = str(_render_raw(node_raw or "", "services.tor.node") or "").strip() group_names = variables.get("group_names") or [] if node_onion and "svc-net-tor" in group_names: - merged = _inject_onion_domains(merged, apps, str(primary_domain), node_onion) + merged = _inject_onion_domains( + merged, + apps, + str(primary_domain), + node_onion, + templar=templar, + variables=variables, + ) _MERGED_DOMAINS_CACHE[cache_key] = merged return merged From 9a68f0eaaf9643915c221160b161fef44c48498b Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 11:52:15 +0200 Subject: [PATCH 035/197] fix(sso): read the OIDC gate's flags through the rendering config lookup meta/addons/oidc_login.yml gates on lookup('sso_oidc_plugin'), which the applications render evaluates while it is still in flight. Under that re-entry guard the merged view hands out service flags as their own Jinja text, and bool("{{ ... }}") is True. The lookup read services.sso.enabled and services.ldap.enabled as enabled on a deployment with neither Keycloak nor OpenLDAP and announced oidc_login, so Nextcloud advertised a login hand-off to a provider that is not there. lookup('config', ...) renders the value it returns, which is why the roughly ninety '| bool' sites in meta/ do not have this problem. 238bc285c7 established reaching the merged config through the lookup loader; this one had not been converted. utils.roles.applications.config.get is the raw accessor and does not render. Measured against the real render: with only web-app-nextcloud deployed the addon set is now empty; adding Keycloak yields sociallogin; adding OpenLDAP as well yields oidc_login. That is the resolution order the lookup's own docstring documents. All 14 nextcloud variants pin both flags as literal booleans, so no CI-deployed variant changes. The stub next to the plugin answered every lookup with the applications payload; it now mirrors lookup('config', app, path, default). A stub resolves a path but does not render it, so it cannot see this defect at all: the regression test renders for real. Co-Authored-By: Claude Opus 5 (1M context) --- plugins/lookup/sso_oidc_plugin.py | 32 ++------- .../plugins/lookup/test_sso_oidc_plugin.py | 25 ++++++- .../test_oidc_addon_follows_sso.py | 68 +++++++++++++++++++ 3 files changed, 97 insertions(+), 28 deletions(-) create mode 100644 tests/unit/python/roles/web-app-nextcloud/test_oidc_addon_follows_sso.py diff --git a/plugins/lookup/sso_oidc_plugin.py b/plugins/lookup/sso_oidc_plugin.py index 3e895dffc2..50c293009c 100644 --- a/plugins/lookup/sso_oidc_plugin.py +++ b/plugins/lookup/sso_oidc_plugin.py @@ -6,9 +6,6 @@ from ansible.errors import AnsibleError from ansible.plugins.loader import lookup_loader from ansible.plugins.lookup import LookupBase -from utils.roles.applications.config import get -from utils.roles.applications.services.sso import get_sso_config - _APPLICATION_ID = "web-app-nextcloud" @@ -40,33 +37,18 @@ class LookupModule(LookupBase): templar = getattr(self, "_templar", None) variables = variables or getattr(self._templar, "available_variables", {}) or {} - applications = lookup_loader.get( - "applications", loader=self._loader, templar=templar - ).run([], variables=variables)[0] + config = lookup_loader.get("config", loader=self._loader, templar=templar) - if not get_sso_config(applications, _APPLICATION_ID)["is_enabled"]: + def _setting(path: str, default: Any) -> Any: + return config.run([_APPLICATION_ID, path, default], variables=variables)[0] + + if not bool(_setting("services.sso.enabled", False)): return [""] - explicit = get( - applications=applications, - application_id=_APPLICATION_ID, - config_path="services.sso.oidc.plugin", - strict=False, - default=None, - skip_missing_app=True, - ) + explicit = _setting("services.sso.oidc.plugin", None) if isinstance(explicit, str) and explicit.strip(): return [explicit.strip()] - ldap_enabled = bool( - get( - applications=applications, - application_id=_APPLICATION_ID, - config_path="services.ldap.enabled", - strict=False, - default=False, - skip_missing_app=True, - ) - ) + ldap_enabled = bool(_setting("services.ldap.enabled", False)) return ["oidc_login" if ldap_enabled else "sociallogin"] diff --git a/tests/unit/python/plugins/lookup/test_sso_oidc_plugin.py b/tests/unit/python/plugins/lookup/test_sso_oidc_plugin.py index 8f6dc8c580..af94622a39 100644 --- a/tests/unit/python/plugins/lookup/test_sso_oidc_plugin.py +++ b/tests/unit/python/plugins/lookup/test_sso_oidc_plugin.py @@ -66,9 +66,28 @@ class OidcFlavorLookupTests(unittest.TestCase): self._patcher = mock.patch.object(self.mod, "lookup_loader") self._loader_mock = self._patcher.start() self._stub_payload = None - self._loader_mock.get.return_value = mock.MagicMock( - run=lambda *_a, **_k: [self._stub_payload] - ) + self._loader_mock.get.side_effect = self._stub_lookup + + def _stub_lookup(self, name, *_args, **_kwargs): + """Stand in for the lookups the plugin consumes. + + ``config`` resolves a dotted path against the payload the way + ``lookup('config', app, path, default)`` does with a default supplied; + anything else answers with the payload itself. + """ + if name == "config": + return mock.MagicMock( + run=lambda terms, **_kw: [self._stub_config(*terms)], + ) + return mock.MagicMock(run=lambda *_a, **_kw: [self._stub_payload]) + + def _stub_config(self, application_id, config_path, default=None): + node = (self._stub_payload or {}).get(application_id) + for part in str(config_path).split("."): + if not isinstance(node, dict) or part not in node: + return default + node = node[part] + return node def tearDown(self): self._patcher.stop() diff --git a/tests/unit/python/roles/web-app-nextcloud/test_oidc_addon_follows_sso.py b/tests/unit/python/roles/web-app-nextcloud/test_oidc_addon_follows_sso.py new file mode 100644 index 0000000000..3fa9e82bee --- /dev/null +++ b/tests/unit/python/roles/web-app-nextcloud/test_oidc_addon_follows_sso.py @@ -0,0 +1,68 @@ +"""The OIDC addon must not be on when Nextcloud has no SSO provider. + +``meta/addons/oidc_login.yml`` gates on ``lookup('sso_oidc_plugin')``, which the +applications render evaluates while it is still in flight. Under that re-entry +guard the merged view hands out service flags as their own Jinja text, and +``bool("{{ ... }}")`` is True: the lookup read ``services.ldap.enabled`` as +enabled on a deployment that has neither Keycloak nor OpenLDAP and announced +``oidc_login``. Nextcloud then advertised a login hand-off to a provider that is +not deployed, while ``templates/config/oidc.config.php.j2`` re-read the same +setting at task time and correctly wrote nothing. + +The stubbed lookup tests next to the plugin cannot see this: a stub resolves the +path but does not render it. This renders for real. +""" + +from __future__ import annotations + +import unittest + +from utils.cache import _reset_cache_for_tests +from utils.cache.applications import get_merged_applications + + +def _templar(variables: dict): + from ansible.parsing.dataloader import DataLoader + from ansible.template import Templar + + templar = Templar(loader=DataLoader(), variables=dict(variables)) + templar.available_variables = dict(variables) + return templar + + +def _addon_flags(*deployed: str) -> dict: + variables = {"group_names": list(deployed), "applications": {}} + _reset_cache_for_tests() + try: + applications = get_merged_applications( + variables=variables, templar=_templar(variables) + ) + finally: + _reset_cache_for_tests() + nextcloud = applications["web-app-nextcloud"] + addons = nextcloud.get("addons") or {} + services = nextcloud.get("services") or {} + return { + "sso": (services.get("sso") or {}).get("enabled"), + "ldap": (services.get("ldap") or {}).get("enabled"), + **{ + name: (addons.get(name) or {}).get("enabled") + for name in ("oidc_login", "user_oidc", "sociallogin") + }, + } + + +class TestOidcAddonFollowsSso(unittest.TestCase): + def test_no_provider_means_no_oidc_addon(self) -> None: + flags = _addon_flags("web-app-nextcloud") + + self.assertEqual(flags["sso"], False) + self.assertEqual( + (flags["oidc_login"], flags["user_oidc"], flags["sociallogin"]), + (False, False, False), + "no OIDC plugin may be announced without an SSO provider", + ) + + +if __name__ == "__main__": + unittest.main() From d7efffc31a6bf044b1a0b2de6143b424d0af649c Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 13:39:35 +0200 Subject: [PATCH 036/197] fix(tor_ports): forward only the ports a role names under ports.onion The public-port sweep never forwarded anything. It read tor.enabled from meta/services.yml on disk, where 104 of 105 roles declare it as the Jinja string "{{ 'svc-net-tor' in group_names }}", which _as_bool answers False for. Running HEAD's collect_public_ports the way HEAD's run() calls it, against the real roles directory with every role deployed, returns []. Only svc-net-tor reads True on disk, and it declares no public ports. The node onion therefore carried port 80 and port 22 and nothing else. Reviving the sweep by reading the merged view instead would have been worse than leaving it dead: it publishes 16 ports across 7 roles, and declared is not published. openldap's ldaps 636 sits next to network.public false, Mailu's implicit-TLS ports are dropped by container_ports whenever TLS is off (which an onion deployment always is), and svc-net-tor's own SOCKS port would become an open proxy. So the source and the selection rule change together: read the variant-merged applications view, and forward only the categories a service names under ports.onion, resolving the number from ports.local when declared there and ports.public otherwise, which is the precedence container_ports publishes by. collect_public_ports had no consumer outside this module and its own test. UDP_ONLY_CATEGORIES becomes public so the accompanying lint imports the list rather than restating it, and gains stun_turn and stun_turn_tls: those speak TCP, but the allocation TURN hands back is a UDP relay address an onion client can never receive on. No role declares ports.onion yet, so the rendered HiddenServicePort set is unchanged by this commit alone. Co-Authored-By: Claude Opus 5 (1M context) --- plugins/lookup/tor_ports.py | 97 +++++---- .../python/plugins/lookup/test_tor_ports.py | 191 +++++++++++++++--- 2 files changed, 223 insertions(+), 65 deletions(-) diff --git a/plugins/lookup/tor_ports.py b/plugins/lookup/tor_ports.py index ec24f33741..68f2bd9413 100644 --- a/plugins/lookup/tor_ports.py +++ b/plugins/lookup/tor_ports.py @@ -1,19 +1,18 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from pathlib import Path +from typing import Any from ansible.errors import AnsibleError from ansible.plugins.loader import lookup_loader from ansible.plugins.lookup import LookupBase -from utils.cache.base import _resolve_roles_dir -from utils.cache.yaml import load_yaml_any -from utils.roles.mapping import ROLE_FILE_META_SERVICES +UDP_ONLY_CATEGORIES = frozenset({"relay", "media", "stun_turn", "stun_turn_tls"}) +"""Categories a hidden service cannot carry. -_UDP_ONLY_CATEGORIES = frozenset({"relay", "media"}) +``relay`` and ``media`` are UDP. ``stun_turn`` and its TLS variant do speak TCP, +but the allocation they hand out is a UDP relay address the onion client can +never receive on, so forwarding the signalling port buys nothing. +""" def _as_bool(value: object) -> bool: @@ -24,6 +23,10 @@ def _as_bool(value: object) -> bool: return bool(value) +def _mapping(value: object) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + def _collect_local_tcp_ports(port_categories: Any, into: set[int]) -> None: """Add single-int TCP ports of the loopback-published ``ports.local`` group only. An exposed service publishes just its local plaintext port to @@ -36,37 +39,57 @@ def _collect_local_tcp_ports(port_categories: Any, into: set[int]) -> None: if not isinstance(local, dict): return for category, value in local.items(): - if category in _UDP_ONLY_CATEGORIES: + if category in UDP_ONLY_CATEGORIES: continue if isinstance(value, int): into.add(value) -def collect_public_ports( +def collect_onion_ports( + applications: dict[str, Any], deployed_roles: list[str], - roles_dir: Path, ) -> list[int]: - """Single-int TCP ``ports.public`` entries of the deployed roles, sorted.""" + """Single-int TCP ports a deployed role asks for by name under ``ports.onion``. + + ``ports.onion`` is a category-keyed map of booleans alongside ``internal`` / + ``local`` / ``public``: it names which of a service's already-declared ports + should answer on the node onion, and nothing else. The port number comes + from ``ports.local`` when that category is declared there and from + ``ports.public`` otherwise, which is the precedence ``container_ports`` + publishes by, so the forward always targets the port the service is actually + reachable on. + + Naming the category is what makes this safe. Sweeping ``ports.public`` + wholesale forwards ports the deployment never publishes -- ldaps 636 is + declared next to ``network.public: false``, and Mailu's implicit-TLS ports + are dropped from the publish list whenever TLS is off, which an onion + deployment always is. + """ ports: set[int] = set() - for role in deployed_roles: - services_yml = roles_dir / role / ROLE_FILE_META_SERVICES - if not services_yml.is_file(): + deployed = set(deployed_roles) + if not isinstance(applications, dict): + return [] + for app_id, cfg in applications.items(): + if app_id not in deployed: continue - data = load_yaml_any(str(services_yml), default_if_missing={}) - if not isinstance(data, dict): + services = (cfg or {}).get("services") if isinstance(cfg, dict) else None + if not isinstance(services, dict): continue - tor = data.get("tor") - if not (isinstance(tor, dict) and _as_bool(tor.get("enabled"))): - continue - for entity in data.values(): + for entity in services.values(): if not isinstance(entity, dict): continue - public = (entity.get("ports") or {}).get("public") - if not isinstance(public, dict): + declared = entity.get("ports") + if not isinstance(declared, dict): continue - for category, value in public.items(): - if category in _UDP_ONLY_CATEGORIES: + wanted = declared.get("onion") + if not isinstance(wanted, dict): + continue + local = _mapping(declared.get("local")) + public = _mapping(declared.get("public")) + for category, flag in wanted.items(): + if category in UDP_ONLY_CATEGORIES or not _as_bool(flag): continue + value = local.get(category, public.get(category)) if isinstance(value, int): ports.add(value) return sorted(ports) @@ -111,16 +134,18 @@ class LookupModule(LookupBase): Returns the HiddenServicePort mappings for the roles in the current deploy (``group_names``): a list of ``{'onion_port': , 'target': - '127.0.0.1:'}`` dicts, sorted by port. Two sources are unioned: + '127.0.0.1:'}`` dicts, sorted by port. Two opt-ins are unioned: - * every ``ports.public`` TCP port of tor-enabled roles, and - * every port of a service that opts in with ``exposed: true`` (resolved - against the variant-merged applications view, so per-variant). + * every port a service names under ``ports.onion`` in meta/services.yml, + and + * every port of a service that opts in with ``exposed: true``. - Public/exposed ports bind the host interface, so the loopback target reaches - them from svc-net-tor's host-network container. Consumed by - ``TOR_ONION_EXTRA_PORTS`` (group_vars/all/19_tor.yml) and rendered as - ``HiddenServicePort`` lines in svc-net-tor's torrc. + Both are read from the variant-merged applications view, so a variant that + drops the opt-in drops the forward with it. The published port binds the + host interface, so the loopback target reaches it from svc-net-tor's + host-network container. Consumed by ``TOR_ONION_EXTRA_PORTS`` + (group_vars/all/19_tor.yml) and rendered as ``HiddenServicePort`` lines in + svc-net-tor's torrc. """ def run( @@ -138,17 +163,15 @@ class LookupModule(LookupBase): group_names = [] deployed = [str(g) for g in group_names] - roles_dir = _resolve_roles_dir(roles_dir=kwargs.get("roles_dir")) - ports = set(collect_public_ports(deployed, roles_dir)) - try: applications = lookup_loader.get( "applications", loader=getattr(self, "_loader", None), templar=getattr(self, "_templar", None), ).run([], variables=variables, roles_dir=kwargs.get("roles_dir"))[0] - except Exception: # noqa: BLE001 no merged view -> exposed ports simply unavailable + except Exception: # noqa: BLE001 no merged view -> no ports to forward applications = {} + ports = set(collect_onion_ports(applications, deployed)) ports.update(collect_exposed_ports(applications, deployed)) return [ diff --git a/tests/unit/python/plugins/lookup/test_tor_ports.py b/tests/unit/python/plugins/lookup/test_tor_ports.py index 5be68ffe0a..67b7a5ad72 100644 --- a/tests/unit/python/plugins/lookup/test_tor_ports.py +++ b/tests/unit/python/plugins/lookup/test_tor_ports.py @@ -9,7 +9,7 @@ from ansible.errors import AnsibleError from plugins.lookup.tor_ports import ( LookupModule, collect_exposed_ports, - collect_public_ports, + collect_onion_ports, ) from utils.cache import _reset_cache_for_tests from utils.cache.yaml import dump_yaml_str @@ -33,7 +33,15 @@ class TestTorPortsLookup(unittest.TestCase): _write_services( self.roles_dir, "web-app-gitea", - {"gitea": {"ports": {"public": {"ssh": 2201}, "local": {"http": 8002}}}}, + { + "gitea": { + "ports": { + "public": {"ssh": 2201}, + "local": {"http": 8002}, + "onion": {"ssh": True}, + } + } + }, ) _write_services( self.roles_dir, @@ -68,40 +76,159 @@ class TestTorPortsLookup(unittest.TestCase): _reset_cache_for_tests() self._tmpdir.cleanup() - def test_collects_public_single_ints_sorted(self) -> None: - ports = collect_public_ports( - ["web-app-gitea", "svc-db-openldap"], self.roles_dir + @staticmethod + def _apps(**roles: dict) -> dict: + """The variant-merged applications view for the given roles. + + Every entity keeps its own ``ports``; the top-level ``tor:`` block of + ``meta/services.yml`` merges in as one more entity alongside them. + """ + return {role: {"services": dict(entities)} for role, entities in roles.items()} + + def test_only_named_categories_are_forwarded_sorted(self) -> None: + apps = self._apps( + **{ + "web-app-gitea": { + "gitea": { + "ports": {"public": {"ssh": 2201}, "onion": {"ssh": True}} + } + }, + "web-app-mailu": { + "mailu": { + "ports": { + "public": {"smtp": 25, "smtps": 465, "imap": 143}, + "onion": {"smtp": True}, + } + } + }, + } ) - self.assertEqual(ports, [636, 2201]) - def test_relay_ranges_excluded(self) -> None: - ports = collect_public_ports(["web-svc-coturn"], self.roles_dir) - self.assertEqual(ports, [3481, 5351]) + ports = collect_onion_ports(apps, ["web-app-gitea", "web-app-mailu"]) - def test_media_category_excluded(self) -> None: - _write_services( - self.roles_dir, - "web-app-jitsi", - {"jitsi": {"ports": {"public": {"media": 10000}, "local": {"http": 8084}}}}, + self.assertEqual(ports, [25, 2201]) + + def test_local_wins_over_public_for_the_same_category(self) -> None: + """``container_ports`` publishes the local port when the category has + one, so the forward must target that and not the public alias.""" + apps = self._apps( + **{ + "web-app-x": { + "x": { + "ports": { + "local": {"http": 8080}, + "public": {"http": 80}, + "onion": {"http": True}, + } + } + } + } ) - self.assertEqual(collect_public_ports(["web-app-jitsi"], self.roles_dir), []) - def test_roles_without_public_ports_yield_nothing(self) -> None: - self.assertEqual(collect_public_ports(["web-app-hugo"], self.roles_dir), []) + self.assertEqual(collect_onion_ports(apps, ["web-app-x"]), [8080]) - def test_tor_disabled_role_excluded(self) -> None: - _write_services( - self.roles_dir, - "web-app-optout", - { - "optout": {"ports": {"public": {"ssh": 2299}}}, - "tor": {"enabled": False}, - }, + def test_a_falsy_or_unrendered_flag_forwards_nothing(self) -> None: + """The input must be the rendered view: template text left in the flag + is not quietly read as enabled.""" + apps = self._apps( + **{ + "web-app-a": { + "a": {"ports": {"public": {"ssh": 22}, "onion": {"ssh": False}}} + }, + "web-app-b": { + "b": { + "ports": { + "public": {"ssh": 23}, + "onion": {"ssh": "{{ 'svc-net-tor' in group_names }}"}, + } + } + }, + } ) - self.assertEqual(collect_public_ports(["web-app-optout"], self.roles_dir), []) - def test_unknown_role_ignored(self) -> None: - self.assertEqual(collect_public_ports(["web-app-nope"], self.roles_dir), []) + self.assertEqual(collect_onion_ports(apps, ["web-app-a", "web-app-b"]), []) + + def test_udp_only_categories_are_never_forwarded(self) -> None: + apps = self._apps( + **{ + "web-svc-coturn": { + "coturn": { + "ports": { + "public": { + "media": 10000, + "stun_turn": 3481, + "stun_turn_tls": 5351, + "ssh": 2299, + "relay": {"start": 20000, "end": 39999}, + }, + "onion": { + "media": True, + "relay": True, + "stun_turn": True, + "stun_turn_tls": True, + "ssh": True, + }, + } + } + } + } + ) + + self.assertEqual(collect_onion_ports(apps, ["web-svc-coturn"]), [2299]) + + def test_a_category_with_no_declared_port_is_skipped(self) -> None: + apps = self._apps( + **{ + "web-app-x": { + "x": {"ports": {"public": {"ssh": 2201}, "onion": {"typo": True}}} + } + } + ) + + self.assertEqual(collect_onion_ports(apps, ["web-app-x"]), []) + + def test_a_role_without_an_onion_block_forwards_nothing(self) -> None: + apps = self._apps( + **{"web-app-hugo": {"hugo": {"ports": {"local": {"http": 8008}}}}} + ) + + self.assertEqual(collect_onion_ports(apps, ["web-app-hugo"]), []) + + def test_the_port_flag_is_the_gate_not_the_tor_flag(self) -> None: + """``services.tor.enabled`` is the dependency edge onto svc-net-tor, not + the port forward; ``ports.onion`` alone decides what gets a + HiddenServicePort.""" + apps = self._apps( + **{ + "web-app-gitea": { + "tor": {"enabled": False}, + "gitea": { + "ports": {"public": {"ssh": 2201}, "onion": {"ssh": True}} + }, + } + } + ) + + self.assertEqual(collect_onion_ports(apps, ["web-app-gitea"]), [2201]) + + def test_role_outside_the_deploy_is_ignored(self) -> None: + apps = self._apps( + **{ + "web-app-gitea": { + "gitea": { + "ports": {"public": {"ssh": 2201}, "onion": {"ssh": True}} + } + } + } + ) + + self.assertEqual(collect_onion_ports(apps, ["web-app-nope"]), []) + + def test_collect_onion_ports_ignores_non_mappings(self) -> None: + self.assertEqual(collect_onion_ports({}, ["a"]), []) + self.assertEqual(collect_onion_ports(None, ["a"]), []) + apps = {"a": {"services": {"s": {"ports": {"onion": True}}}}} + self.assertEqual(collect_onion_ports(apps, ["a"]), []) def test_collect_exposed_ports_gated_and_deployed(self) -> None: apps = { @@ -151,6 +278,14 @@ class TestTorPortsLookup(unittest.TestCase): )[0] self.assertEqual(result, [{"onion_port": 2201, "target": "127.0.0.1:2201"}]) + def test_run_forwards_nothing_without_an_onion_opt_in(self) -> None: + result = self.lookup.run( + [], + variables={"group_names": ["svc-db-openldap", "web-svc-coturn"]}, + roles_dir=str(self.roles_dir), + )[0] + self.assertEqual(result, []) + def test_empty_group_names(self) -> None: result = self.lookup.run([], variables={}, roles_dir=str(self.roles_dir))[0] self.assertEqual(result, []) From dd05caa34f3c9b6d2cbe231d8866dd3fedcdffe6 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 14:51:59 +0200 Subject: [PATCH 037/197] feat(services): require an explicit onion decision per host-bound port A port under ports.local or ports.public binds an address on the stack host, which is what a HiddenServicePort can forward to. Whether it should be forwarded has a cost in both directions: silently off leaves a service unreachable that the operator believes is on the onion, and silently on publishes a database, an admin interface or a SOCKS proxy to everyone holding the onion address. Neither belongs in a default, so the flag is now mandatory and every false carries the reason it is false. The first run produced 165 findings, of which 131 were one architectural fact repeated rather than 131 decisions. Four groups are therefore settled once in the lint instead: the proxy-fronted http, sso and websocket, because the node onion already forwards port 80 to the reverse proxy that routes every vhost, oauth2 hop and socket upgrade behind it; the implicit-TLS variants, because Tor is already the transport encryption and container_ports drops those listeners when TLS is off, which an onion deployment always is; the categories a hidden service cannot carry, listed once in plugins.lookup.tor_ports.UDP_ONLY_CATEGORIES, which the lint imports rather than restates; and any entity that already answers the question through exposed:. That leaves 23 real per-port decisions, each with a reason naming its own consumer. svc-net-tor keeps its own socks, trans and dns off the onion: a published SOCKS port is an open proxy, and SocksPolicy cannot refuse it because the forward arrives from the node's own address. Mailu turns smtp, submission, imap and pop3 on, which is what makes sending and receiving work over the onion, and leaves sieve off because ManageSieve manages filters rather than mail. Measured end to end through the merged applications view, across all 105 roles the deploy now forwards exactly 25, 110, 143 and 587. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/testing/suppression.md | 1 + .../design/role/services/layout.md | 31 +++ roles/svc-db-elasticsearch/meta/services.yml | 3 + roles/svc-db-memcached/meta/services.yml | 3 + roles/svc-db-qdrant/meta/services.yml | 3 + roles/svc-db-rabbitmq/meta/services.yml | 3 + roles/svc-db-redis/meta/services.yml | 3 + roles/svc-db-typesense/meta/services.yml | 3 + roles/svc-dns-unbound/meta/services.yml | 3 + roles/svc-net-tor/meta/services.yml | 7 + roles/svc-registry-cache/meta/services.yml | 3 + roles/svc-registry-docker/meta/services.yml | 3 + roles/web-app-checkmk/meta/services.yml | 3 + roles/web-app-gitea/meta/services.yml | 3 + roles/web-app-gitlab/meta/services.yml | 3 + roles/web-app-mailu/meta/services.yml | 7 + roles/web-app-matrix/meta/services.yml | 3 + roles/web-app-peertube/meta/services.yml | 3 + roles/web-svc-xmpp/meta/services.yml | 5 + .../ansible/services/test_onion_port_flag.py | 226 ++++++++++++++++++ 20 files changed, 319 insertions(+) create mode 100644 tests/lint/ansible/services/test_onion_port_flag.py diff --git a/docs/contributing/actions/testing/suppression.md b/docs/contributing/actions/testing/suppression.md index 8746107ca3..40bbe4747f 100644 --- a/docs/contributing/actions/testing/suppression.md +++ b/docs/contributing/actions/testing/suppression.md @@ -44,6 +44,7 @@ The placement rule is per check. The catalog "Position" column uses these labels | Rule | Position | Affected test | Effect | | --------------------------- | ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `url` | same or above | [test_urls_reachable.py](../../../../tests/external/repository/test_urls_reachable.py) | Skips probing the literal HTTP(S) URL. Use for CDN roots and API base URLs that 4xx without a path. | +| `onion-flag` | same or above | [test_onion_port_flag.py](../../../../tests/lint/ansible/services/test_onion_port_flag.py) | Accepts a `false` under `ports.onion`. The marker MUST carry a reason naming why that port stays off the node onion; a bare marker is rejected. | | `docker-version` | same or above | [test_image_versions.py](../../../../tests/external/update/docker/test_image_versions.py), [test_semver_pinning.py](../../../../tests/lint/ansible/services/test_semver_pinning.py) | Skips both the live version-update warning and the non-semver-pin lint for that image's `version:` key. | | `repository-version` | same or above | [test_repository_versions.py](../../../../tests/external/update/repository/test_repository_versions.py), [test_semver_pinning.py](../../../../tests/lint/ansible/services/test_semver_pinning.py) | Skips both the live ref-update warning and the non-semver-pin lint for that entity's `ref:` key. | | `dockerfile-arg-default` | same or above | [test_arg_without_default.py](../../../../tests/lint/docker/dockerfile/test_arg_without_default.py) | Allows the marked `ARG` to carry a default. The caller normally supplies every build argument so a missing one fails the build instead of selecting a base image nobody chose. | diff --git a/docs/contributing/design/role/services/layout.md b/docs/contributing/design/role/services/layout.md index 75c8677b68..809303a385 100644 --- a/docs/contributing/design/role/services/layout.md +++ b/docs/contributing/design/role/services/layout.md @@ -213,6 +213,8 @@ All port data lives under `.ports` in `meta/services.yml` (no `ports:` s relay: # for port-ranges (coturn, BBB, nextcloud TURN) start: end: + onion: + : # forward this category over the node onion ``` ### `internal` / `local` / `public` Split ๐Ÿงญ @@ -222,6 +224,7 @@ All port data lives under `.ports` in `meta/services.yml` (no `ports:` s | `internal` | **Internal container port.** Lives inside the container's network namespace, addressed by other containers on the same role-local network. NOT a host-bound port. Multiple roles MAY legitimately declare the same value (e.g. several nginx-based apps with `internal: { http: 80 }`). | | `local` | **Localhost-bound host port.** Bound on `127.0.0.1` and only reachable through the front-proxy / SSH tunnels. The OS-level binding namespace is shared across all roles, so `local` values MUST be unique across the whole repo. | | `public` | **Public-facing host port.** Bound on `0.0.0.0` and exposed to the public internet (or to whatever the operator's firewall allows). Same uniqueness rule as `local`. | +| `onion` | **Onion opt-in, not a port.** A category-keyed map of booleans naming which of the entity's own categories get a `HiddenServicePort` on the node onion. It declares no numbers: the port is read from `local` when the category is declared there and from `public` otherwise, matching what `container_ports` publishes. Collision detection ignores it. | ### Always Category-Keyed Maps ๐Ÿ—‚๏ธ @@ -256,6 +259,34 @@ coturn: end: 39999 ``` +### `onion` Forwarding Opt-In ๐Ÿง… + +`ports.onion` names the categories that answer on the node onion. Only the named +categories are forwarded. + +The flag is mandatory, enforced by [test_onion_port_flag.py](../../../../../tests/lint/ansible/services/test_onion_port_flag.py): +every category declared under `local` or `public` needs a `true` or a `false`, +and every `false` needs `# nocheck: onion-flag` plus the reason it is false. +Four groups are settled centrally instead and take no flag: the categories a +hidden service cannot carry (`relay`, `media`, `stun_turn`, `stun_turn_tls`), +the proxy-fronted `http`, `sso` and `websocket`, the implicit-TLS variants +(`smtps`, `imaps`, `pop3s`, `ldaps`), and any entity that already answers the +question through `exposed:`. + +```yaml +mailu: + ports: + public: + smtp: 25 + smtps: 465 + imap: 143 + onion: + smtp: true # 25 gets a HiddenServicePort; 465 and 143 do not +``` + +The forward is per-variant: `meta/variants.yml` may drop the flag, and the role +then loses the `HiddenServicePort` in that variant. + ### Multi-Entity Roles ๐ŸŽ›๏ธ Each entity carries its own `ports` block: diff --git a/roles/svc-db-elasticsearch/meta/services.yml b/roles/svc-db-elasticsearch/meta/services.yml index 64c6d96b81..6f13b566e6 100644 --- a/roles/svc-db-elasticsearch/meta/services.yml +++ b/roles/svc-db-elasticsearch/meta/services.yml @@ -25,6 +25,9 @@ elasticsearch: elasticsearch: 9200 internal: elasticsearch: 9200 + onion: + # nocheck: onion-flag the index is queried by the search backends of the apps sharing this stack, never by a client off the node + elasticsearch: false lifecycle: beta # nocheck: variants-coverage โ€” sso-variant-tor requires the onion on in every variant, so the false polarity has no variant left to live in tor: diff --git a/roles/svc-db-memcached/meta/services.yml b/roles/svc-db-memcached/meta/services.yml index d47db749a6..1cd0395f56 100644 --- a/roles/svc-db-memcached/meta/services.yml +++ b/roles/svc-db-memcached/meta/services.yml @@ -28,6 +28,9 @@ memcached: memcached: 11211 internal: memcached: 11211 + onion: + # nocheck: onion-flag the session cache speaks an unauthenticated text protocol and is read only by app containers on the role network + memcached: false lifecycle: beta # nocheck: variants-coverage โ€” sso-variant-tor requires the onion on in every variant, so the false polarity has no variant left to live in tor: diff --git a/roles/svc-db-qdrant/meta/services.yml b/roles/svc-db-qdrant/meta/services.yml index fd8ee4690d..50a5a5fb7c 100644 --- a/roles/svc-db-qdrant/meta/services.yml +++ b/roles/svc-db-qdrant/meta/services.yml @@ -28,6 +28,9 @@ qdrant: qdrant: 6333 internal: qdrant: 6333 + onion: + # nocheck: onion-flag the vector store is read by the svc-ai roles in this stack; its MCP sidecar is the surface meant for outside callers + qdrant: false lifecycle: beta # nocheck: playwright-service-flag internal adapter sidecar, covered by the role's MCP integration test # nocheck: dockerfile-custom the Dockerfile is owned by svc-ai-mcp-adapter and staged per instance, so every adapter-backed provider shares one audited build diff --git a/roles/svc-db-rabbitmq/meta/services.yml b/roles/svc-db-rabbitmq/meta/services.yml index 330ec8ef2c..b3183f721d 100644 --- a/roles/svc-db-rabbitmq/meta/services.yml +++ b/roles/svc-db-rabbitmq/meta/services.yml @@ -28,6 +28,9 @@ rabbitmq: rabbitmq: 5672 internal: rabbitmq: 5672 + onion: + # nocheck: onion-flag AMQP carries the apps' own worker queues; a broker reachable off the node would let anyone enqueue jobs + rabbitmq: false lifecycle: beta # nocheck: variants-coverage โ€” sso-variant-tor requires the onion on in every variant, so the false polarity has no variant left to live in tor: diff --git a/roles/svc-db-redis/meta/services.yml b/roles/svc-db-redis/meta/services.yml index b22b73508c..6abd7afc9a 100644 --- a/roles/svc-db-redis/meta/services.yml +++ b/roles/svc-db-redis/meta/services.yml @@ -25,6 +25,9 @@ redis: redis: 6379 internal: redis: 6379 + onion: + # nocheck: onion-flag the shared cache and queue runs without AUTH inside the stack, so its reachability must stay bounded by the role network + redis: false lifecycle: beta # nocheck: variants-coverage โ€” sso-variant-tor requires the onion on in every variant, so the false polarity has no variant left to live in tor: diff --git a/roles/svc-db-typesense/meta/services.yml b/roles/svc-db-typesense/meta/services.yml index fe07bdae91..8304c9e73a 100644 --- a/roles/svc-db-typesense/meta/services.yml +++ b/roles/svc-db-typesense/meta/services.yml @@ -28,6 +28,9 @@ typesense: typesense: 8108 internal: typesense: 8108 + onion: + # nocheck: onion-flag the search index is queried by the app containers holding its API key, and the apps themselves are served through the onion's port 80 + typesense: false lifecycle: beta # nocheck: variants-coverage โ€” sso-variant-tor requires the onion on in every variant, so the false polarity has no variant left to live in tor: diff --git a/roles/svc-dns-unbound/meta/services.yml b/roles/svc-dns-unbound/meta/services.yml index bb8b51983d..efc6433bcf 100644 --- a/roles/svc-dns-unbound/meta/services.yml +++ b/roles/svc-dns-unbound/meta/services.yml @@ -23,6 +23,9 @@ unbound: ports: local: unbound: 53 + onion: + # nocheck: onion-flag a recursive resolver published on the onion is an open resolver, and the containers it serves reach it on the role network + unbound: false lifecycle: beta # nocheck: dynamic-flag - the resolver state volume is backup-disabled rebuildable cache, the backup consumer stays permanently off diff --git a/roles/svc-net-tor/meta/services.yml b/roles/svc-net-tor/meta/services.yml index 5ffe165550..c28138c505 100644 --- a/roles/svc-net-tor/meta/services.yml +++ b/roles/svc-net-tor/meta/services.yml @@ -34,6 +34,13 @@ tor: # nocheck: category-modes socks: 9050 trans: 9040 dns: 9053 + onion: + # nocheck: onion-flag a SOCKS port anyone can reach is an open proxy, and SocksPolicy cannot refuse it because the forward arrives from the node's own address + socks: false + # nocheck: onion-flag the transparent port takes its destination from the caller's routing, which an onion client does not share + trans: false + # nocheck: onion-flag publishing the resolver would answer recursive queries for anyone holding the onion address + dns: false min_storage: 100MB cpus: "0.25" mem_reservation: 128m diff --git a/roles/svc-registry-cache/meta/services.yml b/roles/svc-registry-cache/meta/services.yml index 582e5433bd..69a32710b8 100644 --- a/roles/svc-registry-cache/meta/services.yml +++ b/roles/svc-registry-cache/meta/services.yml @@ -34,6 +34,9 @@ cache: registry: 5001 internal: registry: 5000 + onion: + # nocheck: onion-flag the pull-through mirror exists to serve this node's own image pulls; publishing it would relay Docker Hub for strangers + registry: false proxy: remote_url: "https://registry-1.docker.io" # nocheck: url lifecycle: beta diff --git a/roles/svc-registry-docker/meta/services.yml b/roles/svc-registry-docker/meta/services.yml index 5b2d65525d..66774e620e 100644 --- a/roles/svc-registry-docker/meta/services.yml +++ b/roles/svc-registry-docker/meta/services.yml @@ -34,6 +34,9 @@ docker: registry: 5000 internal: registry: 5000 + onion: + # nocheck: onion-flag the registry accepts unauthenticated pushes from the node that builds into it, so its reach must stay the node + registry: false lifecycle: beta # nocheck: dynamic-flag - locally built images are rebuildable, the backup consumer stays permanently off container_backup: diff --git a/roles/web-app-checkmk/meta/services.yml b/roles/web-app-checkmk/meta/services.yml index 16a28d1907..ab52dec53c 100644 --- a/roles/web-app-checkmk/meta/services.yml +++ b/roles/web-app-checkmk/meta/services.yml @@ -83,6 +83,9 @@ checkmk: http: 8087 sso: 16509 agent: 8500 + onion: + # nocheck: onion-flag the monitoring server dials its agents, so nothing connects inward here, and the agent speaks no authentication of its own + agent: false run_after: - web-app-keycloak - svc-db-openldap diff --git a/roles/web-app-gitea/meta/services.yml b/roles/web-app-gitea/meta/services.yml index 751acb0134..d9d25c94e3 100644 --- a/roles/web-app-gitea/meta/services.yml +++ b/roles/web-app-gitea/meta/services.yml @@ -76,6 +76,9 @@ gitea: http: 8002 public: ssh: 2201 + onion: + # nocheck: onion-flag git over the onion runs on the HTTP transport the proxy already serves on port 80; a second sshd on the onion would need its own host key policy + ssh: false run_after: - web-app-matomo - web-app-keycloak diff --git a/roles/web-app-gitlab/meta/services.yml b/roles/web-app-gitlab/meta/services.yml index 0eaf633988..146427958e 100644 --- a/roles/web-app-gitlab/meta/services.yml +++ b/roles/web-app-gitlab/meta/services.yml @@ -63,6 +63,9 @@ gitlab: internal: http: 8181 ssh: 2222 + onion: + # nocheck: onion-flag GitLab Shell is published for clearnet git clients and authenticates them against the internal API by host key; onion clients clone over the HTTP transport on port 80 instead + ssh: false proxy: service_key: workhorse run_after: diff --git a/roles/web-app-mailu/meta/services.yml b/roles/web-app-mailu/meta/services.yml index 99eb7abc1d..6a830b3e08 100644 --- a/roles/web-app-mailu/meta/services.yml +++ b/roles/web-app-mailu/meta/services.yml @@ -61,6 +61,13 @@ mailu: imap: 143 imaps: 993 sieve: 4190 + onion: + smtp: true + submission: true + imap: true + pop3: true + # nocheck: onion-flag ManageSieve edits server-side filters, which is administration rather than the sending and receiving the onion transport is for + sieve: false proxy: service_key: front run_after: diff --git a/roles/web-app-matrix/meta/services.yml b/roles/web-app-matrix/meta/services.yml index 77e592c100..024e4179be 100644 --- a/roles/web-app-matrix/meta/services.yml +++ b/roles/web-app-matrix/meta/services.yml @@ -55,6 +55,9 @@ synapse: http: 8021 public: federation: 8448 + onion: + # nocheck: onion-flag federation is dialled by clearnet homeservers resolving this server_name; serving it on the onion needs an onion server_name, which is a different deployment + federation: false min_storage: 311MB cpus: "1.5" mem_reservation: "1g" diff --git a/roles/web-app-peertube/meta/services.yml b/roles/web-app-peertube/meta/services.yml index 9132986606..504da92f83 100644 --- a/roles/web-app-peertube/meta/services.yml +++ b/roles/web-app-peertube/meta/services.yml @@ -79,6 +79,9 @@ peertube: http: 8011 public: rtmp: 1935 + onion: + # nocheck: onion-flag RTMP is an ingest the broadcaster pushes in, and the encoders that push it speak no SOCKS + rtmp: false internal: http: 9000 rtmp: 1935 diff --git a/roles/web-svc-xmpp/meta/services.yml b/roles/web-svc-xmpp/meta/services.yml index 4fbbdadc6e..6229ec6578 100644 --- a/roles/web-svc-xmpp/meta/services.yml +++ b/roles/web-svc-xmpp/meta/services.yml @@ -38,6 +38,11 @@ xmpp: public: c2s: 5222 s2s: 5269 + onion: + # nocheck: onion-flag onion clients reach the same sessions over the BOSH and websocket endpoints the proxy serves on port 80 + c2s: false + # nocheck: onion-flag server-to-server dials clearnet peers that resolve this domain in DNS, which an onion address does not answer + s2s: false internal: http: 5280 c2s: 5222 diff --git a/tests/lint/ansible/services/test_onion_port_flag.py b/tests/lint/ansible/services/test_onion_port_flag.py new file mode 100644 index 0000000000..6e5fc32ab5 --- /dev/null +++ b/tests/lint/ansible/services/test_onion_port_flag.py @@ -0,0 +1,226 @@ +"""Lint: every host-bound port states whether it answers on the node onion. + +A port under ``ports.local`` or ``ports.public`` binds an address on the stack +host, which is what a ``HiddenServicePort`` can forward to. Whether it should be +forwarded is a decision with a cost in both directions: silently off leaves a +service unreachable that the operator believes is on the onion, and silently on +publishes a database, an admin interface or a SOCKS proxy to everyone holding +the onion address. Neither belongs to a default, so the flag is mandatory and +every ``false`` carries the reason it is false. + +The flag lives next to the numbers it decides about:: + + ports: + public: + smtp: 25 + smtps: 465 + onion: + smtp: true + smtps: false # nocheck: onion-flag no implicit-TLS listener when TLS is off + +Five groups are out of scope, because for them the answer is architectural +rather than per-role, and a hundred-odd copies of one sentence state it worse +than this paragraph does: + +* ``ports.internal`` lives in the container's network namespace, where no + host-side forward reaches it. +* the categories a hidden service cannot carry, listed and reasoned once in + ``plugins.lookup.tor_ports.UDP_ONLY_CATEGORIES``. +* ``http``, ``sso`` and ``websocket``, because the node onion already forwards + port 80 to the reverse proxy, which routes every vhost, every oauth2 hop and + every socket upgrade behind it. A direct forward there would bypass the proxy, + its auth and its CSP. +* the implicit-TLS variants, because Tor is already the transport encryption and + ``container_ports`` drops those listeners entirely when TLS is off, which an + onion deployment always is. +* an entity declaring ``exposed:``, which is the older opt-in for the same + decision and is tested per variant. Two flags for one decision is the trap. + +A flag naming a category the entity never declares is reported too: it reads as +a live forward and forwards nothing. + +Suppression (see ``docs/contributing/actions/testing/suppression.md``): + +* ``# nocheck: onion-flag`` plus a reason, on or directly above a ``false`` flag. +""" + +from __future__ import annotations + +import re +import unittest +from collections.abc import Mapping + +from plugins.lookup.tor_ports import UDP_ONLY_CATEGORIES +from utils.annotations.suppress import line_has_rule +from utils.cache.files import read_text +from utils.cache.yaml import load_yaml_any +from utils.meta.scan import iter_role_dirs +from utils.roles.mapping import ROLE_FILE_META_SERVICES + +_RULE = "onion-flag" +_HOST_BOUND_SCOPES = ("local", "public") +_PROXY_FRONTED_CATEGORIES = frozenset({"http", "sso", "websocket"}) +_IMPLICIT_TLS_CATEGORIES = frozenset({"smtps", "imaps", "pop3s", "ldaps"}) +_OUT_OF_SCOPE = ( + UDP_ONLY_CATEGORIES | _PROXY_FRONTED_CATEGORIES | _IMPLICIT_TLS_CATEGORIES +) +_MIN_REASON_CHARS = 10 + +_MARKER_RE = re.compile( + r"(?:noqa|nocheck)\s*:\s*" + r"(?:[a-z0-9][a-z0-9\-]*)(?:\s*,\s*[a-z0-9][a-z0-9\-]*)*(.*)$", + re.IGNORECASE, +) + + +def _host_bound_categories(ports: Mapping) -> set[str]: + """Category names of *ports* that bind a host address as a single TCP port.""" + found: set[str] = set() + for scope in _HOST_BOUND_SCOPES: + declared = ports.get(scope) + if not isinstance(declared, Mapping): + continue + found.update( + category + for category, value in declared.items() + if isinstance(value, int) + and not isinstance(value, bool) + and category not in _OUT_OF_SCOPE + ) + return found + + +def _flag_line(lines: list[str], entity: str, category: str) -> int | None: + """1-based line of ``:`` inside *entity*'s ``onion:`` block.""" + in_entity = False + onion_indent = -1 + for number, raw in enumerate(lines, start=1): + stripped = raw.strip() + if not stripped or stripped.startswith("#"): + continue + indent = len(raw) - len(raw.lstrip()) + if indent == 0: + in_entity = stripped.startswith(f"{entity}:") + onion_indent = -1 + continue + if not in_entity: + continue + if onion_indent >= 0 and indent <= onion_indent: + onion_indent = -1 + if stripped.startswith("onion:"): + onion_indent = indent + continue + if onion_indent >= 0 and stripped.startswith(f"{category}:"): + return number + return None + + +def _reason_for(lines: list[str], line_no: int) -> str | None: + """The suppression reason governing *line_no*, or None when unsuppressed.""" + candidates = [line_no - 1] + previous = line_no - 2 + while previous >= 0 and not lines[previous].strip(): + previous -= 1 + candidates.append(previous) + for index in candidates: + if index < 0 or not line_has_rule(lines[index], _RULE): + continue + match = _MARKER_RE.search(lines[index]) + return match.group(1).strip().lstrip("-:").strip() if match else "" + return None + + +class TestOnionPortFlag(unittest.TestCase): + def test_every_host_bound_port_declares_an_onion_flag(self) -> None: + findings: list[str] = [] + for role_dir in iter_role_dirs(): + path = role_dir / ROLE_FILE_META_SERVICES + if not path.is_file(): + continue + services = load_yaml_any(str(path), default_if_missing={}) + if not isinstance(services, Mapping): + continue + lines = read_text(str(path)).splitlines() + for entity, config in services.items(): + if not isinstance(config, Mapping): + continue + ports = config.get("ports") + if not isinstance(ports, Mapping) or "exposed" in config: + continue + findings.extend( + self._entity_findings(role_dir.name, entity, ports, lines) + ) + if findings: + self.fail( + "Every host-bound port must state whether it answers on the node " + "onion, via a boolean of the same category name under " + "`ports.onion`. A `false` flag needs `# nocheck: onion-flag` and " + "a reason on or above it.\n" + "\n".join(findings) + ) + + def _entity_findings( + self, + role: str, + entity: str, + ports: Mapping, + lines: list[str], + ) -> list[str]: + declared = _host_bound_categories(ports) + flags = ports.get("onion") + flags = flags if isinstance(flags, Mapping) else {} + where = f"roles/{role}/{ROLE_FILE_META_SERVICES}: {entity}" + findings = [ + f"{where}.ports.onion.{category}: missing, declare true or false" + for category in sorted(declared - set(flags)) + ] + findings.extend( + f"{where}.ports.onion.{category}: " + + ( + "out of scope, remove the flag" + if category in _OUT_OF_SCOPE + else "no such port is declared under " + " or ".join(_HOST_BOUND_SCOPES) + ) + for category in sorted(set(flags) - declared) + ) + for category in sorted(set(flags) & declared): + findings.extend( + self._flag_findings(where, entity, category, flags[category], lines) + ) + return findings + + def _flag_findings( + self, + where: str, + entity: str, + category: str, + flag: object, + lines: list[str], + ) -> list[str]: + at = f"{where}.ports.onion.{category}" + if not isinstance(flag, bool): + return [f"{at}: must be a literal true or false, not {flag!r}"] + if flag: + return [] + line_no = _flag_line(lines, entity, category) + if line_no is None: + return [f"{at}: is false but its line could not be located"] + reason = _reason_for(lines, line_no) + if reason is None: + return [ + ( + f"{at} (line {line_no}): false needs `# nocheck: {_RULE}` and " + f"a reason on or above it" + ) + ] + if len(reason) < _MIN_REASON_CHARS: + return [ + ( + f"{at} (line {line_no}): `# nocheck: {_RULE}` states no reason " + f"why the port stays off the onion" + ) + ] + return [] + + +if __name__ == "__main__": + unittest.main() From b036f6003978e444c24826e0a4435d535675ad8e Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 14:55:25 +0200 Subject: [PATCH 038/197] refactor(container_ports): express the remaining published ports through the lookup The list form could publish a host:container pair and nothing else, so thirteen blocks stayed hand-written: the ones needing a protocol suffix, a port range, the swarm long syntax with mode host, or a container-port-only publish. Each of those duplicated numbers the role already declares in meta/services.yml, which is the drift the lookup exists to remove. The lookup keeps the list form unchanged for its 93 existing call sites and gains a dict form: transport emits one item per protocol, container 'same' serves the services that cannot remap (STUN/TURN, media relays), mode 'host' emits the long block an edge role needs in swarm so the client address survives the hop, publish 'ephemeral' emits the container port alone, and key names the block for Discourse's launcher file, whose expose: list is what becomes docker run -p there. A declared {start, end} mapping renders as start-end on both sides, so a relay range needs no term of its own. svc-prx-openresty had no ports block at all and bound 80 and 443 as literals in its template, while torrc.j2 carried a third literal 80 for the hidden service. None of the three referenced the others. openresty now declares the ports, the compose template reads them, and the onion's port 80 comes from ports.onion.http through TOR_ONION_EXTRA_PORTS. Three blocks stay raw, each with the reason it cannot route through one call: keycloak's sso_proxy maps a published port owned by the consuming role to a container port owned by web-app-keycloak, and matrix's ansible flavor takes element on 8080 and federation on 8048 from the upstream playbook rather than from this repo's ports.internal. The enforcing lint scanned only files with compose in the name and only double-quoted list items, so nine of the thirteen blocks were invisible to it. It now reads every role template that can carry a ports block, catches unquoted items, protocol suffixes, ranges, flow style and the long syntax, and tracks the block by the indentation of its key: keying on the next line closed the block on anything shaped like a key, which target, published, protocol and mode all are. Under expose: only a real mapping counts, because there the single-segment form documents a container port and publishes nothing. Verified by rendering all 110 container_ports call sites through the real templar, the real lookup and the real merged applications view. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/08_networks.yml | 6 + plugins/lookup/container_ports.py | 211 ++++++++++++++---- roles/svc-dns-unbound/meta/services.yml | 2 + .../svc-dns-unbound/templates/compose.yml.j2 | 6 +- roles/svc-dns-unbound/vars/main.yml | 1 - roles/svc-net-tor/templates/compose.yml.j2 | 8 +- roles/svc-net-tor/vars/main.yml | 1 - roles/svc-prx-openresty/meta/services.yml | 11 + .../templates/compose.yml.j2 | 12 +- .../templates/compose.override.yml.j2 | 12 +- roles/web-app-bigbluebutton/vars/main.yml | 1 - .../templates/compose.yml.j2 | 4 +- .../web-app-discourse/templates/config.yml.j2 | 3 +- .../templates/compose.yml.j2 | 5 +- roles/web-app-jitsi/templates/compose.yml.j2 | 5 +- .../templates/sso_proxy/container.yml.j2 | 1 + .../templates/flavor/ansible/services.yml.j2 | 5 +- .../templates/flavor/compose/services.yml.j2 | 8 +- roles/web-app-moodle/templates/compose.yml.j2 | 3 +- .../templates/compose.yml.j2 | 12 +- roles/web-svc-coturn/templates/compose.yml.j2 | 24 +- roles/web-svc-coturn/vars/main.yml | 1 - ...ompose_ports_use_container_ports_lookup.py | 121 ++++++---- .../plugins/lookup/test_container_ports.py | 128 +++++++++++ 24 files changed, 457 insertions(+), 134 deletions(-) diff --git a/group_vars/all/08_networks.yml b/group_vars/all/08_networks.yml index fa1920561c..25f3a66890 100644 --- a/group_vars/all/08_networks.yml +++ b/group_vars/all/08_networks.yml @@ -82,6 +82,12 @@ PORT_BANDS: # nocheck: unused-var # consumed by the port-allocation plugin outs start: 9053 end: 9053 public: + http: + start: 80 + end: 80 + https: + start: 443 + end: 443 ssh: start: 2201 end: 2299 diff --git a/plugins/lookup/container_ports.py b/plugins/lookup/container_ports.py index 87ce236006..f6cb5504a2 100644 --- a/plugins/lookup/container_ports.py +++ b/plugins/lookup/container_ports.py @@ -1,33 +1,67 @@ -"""Lookup `container_ports`: build a compose ``ports:`` publish block from -``[service_name, protocol]`` or ``[service_name, protocol, host_ip]`` terms. +"""Lookup `container_ports`: build a compose ``ports:`` block from declared ports. -Single SPOT replacing the repeated, verbose port lookups. Each term is a -two- or three-element list; pass one or more. The optional third element is -the host IP to bind that port to; omit it to publish on every interface: +Single SPOT for every published port in the repo. A term is either the short +list form or the dict form; pass one or more. + +List form, unchanged:: {{ lookup('container_ports', ['gitea', 'http', DOCKER_BIND_HOST], ['gitea', 'ssh']) | indent(4) }} -produces +produces:: ports: - - "::" + - "::<...internal.http>" - ":" -Per term the published (host) port is ``services..ports.local.`` -when declared, else ``services..ports.public.``; the container -port is always ``services..ports.internal.``. A third element -prefixes ``:``; without it the port binds to all interfaces. This -makes every mapping expressible, including binds to a specific public IP -(e.g. ``['mailu', 'smtp', MAILU_IP4_PUBLIC]``). +The published (host) port is ``ports.local.`` when declared, else +``ports.public.``; the container port is ``ports.internal.``. -``application_id`` is read from the play variables unless given as ``application_id=``. +Dict form, for the shapes the list form cannot express:: + + {{ lookup('container_ports', + {'service': 'coturn', 'protocol': 'stun_turn', + 'transport': ['udp', 'tcp'], 'container': 'same'}, + {'service': 'coturn', 'protocol': 'relay', 'transport': 'udp', + 'container': 'same'}, + {'service': 'openresty', 'protocol': 'http', 'mode': 'host'}, + {'service': 'funkwhale', 'protocol': 'api', + 'publish': 'ephemeral'}) | indent(4) }} + +Keys: + +``service``, ``protocol`` + Required, as in the list form. +``ip`` + Host address to bind to; omit to publish on every interface. +``transport`` + ``tcp``, ``udp``, or a list of both. One list item per transport. Omit for + compose's own default, which publishes no suffix. +``container`` + ``internal`` (default) reads ``ports.internal.``; ``same`` reuses the + published port, for services that cannot remap (STUN/TURN, media relays). +``mode`` + ``ingress`` (default) emits the short ``"host:published:container"`` string; + ``host`` emits the long block with ``mode: host``, which is what an edge + role needs in swarm so the client address survives the hop. +``publish`` + ``ephemeral`` publishes the container port on a host port docker picks, for + a service addressed only through the compose network. + +A declared value that is a ``{start, end}`` mapping renders as ``start-end`` on +both sides, so a relay range needs no special term. + +``application_id`` is read from the play variables unless given as +``application_id=``. ``key='expose'`` names the block instead of ``ports``, for +Discourse's launcher file, whose ``expose:`` list is what becomes ``docker run +-p`` there. """ from __future__ import annotations import contextlib +from collections.abc import Mapping from typing import Any from ansible.errors import AnsibleError @@ -37,11 +71,115 @@ from ansible.template import trust_as_template from utils.roles.applications.config import get +_TERM_KEYS = frozenset( + {"service", "protocol", "ip", "transport", "container", "mode", "publish"} +) +_TRANSPORTS = ("tcp", "udp") +_CONTAINER_SOURCES = ("internal", "same") +_MODES = ("ingress", "host") + def _as_str(value: Any) -> str: return "" if value is None else str(value).strip() +def _rendered(value: Any) -> str: + """A declared port value as compose writes it, ranges included.""" + if isinstance(value, Mapping): + start, end = value.get("start"), value.get("end") + if start is None or end is None: + return "" + return f"{_as_str(start)}-{_as_str(end)}" + return _as_str(value) + + +def _transports_of(term: Mapping) -> list[str]: + declared = term.get("transport") + if declared is None: + return [""] + wanted = [declared] if isinstance(declared, str) else list(declared) + for transport in wanted: + if transport not in _TRANSPORTS: + raise AnsibleError( + f"container_ports: transport must be one of {_TRANSPORTS}, " + f"got {transport!r}" + ) + return wanted + + +def _normalised(term: Any) -> dict[str, Any]: + """One term as a dict, accepting the legacy list form.""" + if isinstance(term, (list, tuple)): + if len(term) not in (2, 3): + raise AnsibleError( + "container_ports: a list term must be [service_name, protocol] " + f"or [service_name, protocol, host_ip], got {term!r}" + ) + term = { + "service": term[0], + "protocol": term[1], + **({"ip": term[2]} if len(term) == 3 else {}), + } + if not isinstance(term, Mapping): + raise AnsibleError( + f"container_ports: each term must be a list or a mapping, got {term!r}" + ) + unknown = set(term) - _TERM_KEYS + if unknown: + raise AnsibleError( + f"container_ports: unknown term key(s) {sorted(unknown)}; " + f"valid keys are {sorted(_TERM_KEYS)}" + ) + normalised = { + "service": _as_str(term.get("service")), + "protocol": _as_str(term.get("protocol")), + "ip": _as_str(term.get("ip")), + "transport": term.get("transport"), + "container": _as_str(term.get("container")) or "internal", + "mode": _as_str(term.get("mode")) or "ingress", + "publish": _as_str(term.get("publish")), + } + if not normalised["service"] or not normalised["protocol"]: + raise AnsibleError("container_ports: service and protocol must be non-empty") + if normalised["container"] not in _CONTAINER_SOURCES: + raise AnsibleError( + f"container_ports: container must be one of {_CONTAINER_SOURCES}, " + f"got {normalised['container']!r}" + ) + if normalised["mode"] not in _MODES: + raise AnsibleError( + f"container_ports: mode must be one of {_MODES}, got {normalised['mode']!r}" + ) + if normalised["publish"] and normalised["publish"] != "ephemeral": + raise AnsibleError( + "container_ports: publish accepts only 'ephemeral', " + f"got {normalised['publish']!r}" + ) + if normalised["mode"] == "host" and normalised["ip"]: + raise AnsibleError( + "container_ports: mode 'host' binds the node's own interface and " + "takes no ip" + ) + return normalised + + +def _short_line(term: Mapping, published: str, container: str, transport: str) -> str: + suffix = f"/{transport}" if transport else "" + if term["publish"] == "ephemeral": + return f' - "{container}{suffix}"' + head = f"{term['ip']}:" if term["ip"] else "" + return f' - "{head}{published}:{container}{suffix}"' + + +def _host_lines(published: str, container: str, transport: str) -> list[str]: + return [ + f" - target: {container}", + f" published: {published}", + f" protocol: {transport or 'tcp'}", + " mode: host", + ] + + class LookupModule(LookupBase): def run( self, @@ -58,22 +196,9 @@ class LookupModule(LookupBase): if not terms: raise AnsibleError( "lookup('container_ports', [service_name, protocol[, host_ip]], ...) " - "expects one or more [service_name, protocol] terms" + "expects one or more terms" ) - triples: list[tuple[str, str, str]] = [] - for term in terms: - if not (isinstance(term, (list, tuple)) and len(term) in (2, 3)): - raise AnsibleError( - "container_ports: each term must be a [service_name, protocol] " - f"or [service_name, protocol, host_ip] list, got {term!r}" - ) - service, protocol = _as_str(term[0]), _as_str(term[1]) - host = _as_str(term[2]) if len(term) == 3 else "" - if not service or not protocol: - raise AnsibleError( - "container_ports: service_name and protocol must be non-empty" - ) - triples.append((service, protocol, host)) + parsed = [_normalised(term) for term in terms] templar = getattr(self, "_templar", None) variables = variables or getattr(self._templar, "available_variables", {}) or {} @@ -96,7 +221,7 @@ class LookupModule(LookupBase): ).run([], variables=variables)[0] def _port(service: str, scope: str, protocol: str, *, required: bool) -> str: - value = _as_str( + value = _rendered( get( applications=applications, application_id=application_id, @@ -112,14 +237,24 @@ class LookupModule(LookupBase): ) return value - lines = ["ports:"] - for service, protocol, host in triples: - internal = _port(service, "internal", protocol, required=True) - published = _port(service, "local", protocol, required=False) or _port( - service, "public", protocol, required=True + block_key = _as_str(kwargs.get("key")) or "ports" + if block_key not in ("ports", "expose"): + raise AnsibleError( + f"container_ports: key must be 'ports' or 'expose', got {block_key!r}" ) - if host: - lines.append(f' - "{host}:{published}:{internal}"') + lines = [f"{block_key}:"] + for term in parsed: + service, protocol = term["service"], term["protocol"] + published = _port(service, "local", protocol, required=False) or _port( + service, "public", protocol, required=term["publish"] != "ephemeral" + ) + if term["container"] == "same": + container = published else: - lines.append(f' - "{published}:{internal}"') + container = _port(service, "internal", protocol, required=True) + for transport in _transports_of(term): + if term["mode"] == "host": + lines.extend(_host_lines(published, container, transport)) + else: + lines.append(_short_line(term, published, container, transport)) return ["\n".join(lines)] diff --git a/roles/svc-dns-unbound/meta/services.yml b/roles/svc-dns-unbound/meta/services.yml index efc6433bcf..e440222e4b 100644 --- a/roles/svc-dns-unbound/meta/services.yml +++ b/roles/svc-dns-unbound/meta/services.yml @@ -21,6 +21,8 @@ unbound: mem_limit: "512m" pids_limit: 256 ports: + internal: + unbound: 53 local: unbound: 53 onion: diff --git a/roles/svc-dns-unbound/templates/compose.yml.j2 b/roles/svc-dns-unbound/templates/compose.yml.j2 index f6371fed79..86f759571d 100644 --- a/roles/svc-dns-unbound/templates/compose.yml.j2 +++ b/roles/svc-dns-unbound/templates/compose.yml.j2 @@ -6,9 +6,9 @@ {{ lookup('container_image', application_id, 'unbound') }} {% include 'roles/sys-svc-container/templates/base.yml.j2' %} {% if DEPLOYMENT_MODE == 'compose' and not (DOCKER_IN_CONTAINER | bool) %} - ports: - - "{{ DOCKER_BIND_HOST }}:{{ UNBOUND_PORT }}:53/udp" # nocheck: compose-ports-must-use-container-ports - - "{{ DOCKER_BIND_HOST }}:{{ UNBOUND_PORT }}:53/tcp" # nocheck: compose-ports-must-use-container-ports + {{ lookup('container_ports', + {'service': 'unbound', 'protocol': 'unbound', 'ip': DOCKER_BIND_HOST, + 'transport': ['udp', 'tcp']}) | indent(4) }} {% endif %} {{ lookup('container_volumes', service_name) | indent(4) }} networks: # nocheck: networks-literal static-IP resolver on its own engine overlay so consumers can point dns: at it diff --git a/roles/svc-dns-unbound/vars/main.yml b/roles/svc-dns-unbound/vars/main.yml index 918bac40d0..065d64f95f 100644 --- a/roles/svc-dns-unbound/vars/main.yml +++ b/roles/svc-dns-unbound/vars/main.yml @@ -4,7 +4,6 @@ entity_name: "{{ application_id | get_entity_name }}" UNBOUND_CONTAINER: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.name') }}" UNBOUND_CONTAINER_ADDRESS: "{{ lookup('container_address', application_id, UNBOUND_CONTAINER) }}" -UNBOUND_PORT: "{{ lookup('config', application_id, 'services.unbound.ports.local.unbound') }}" UNBOUND_NETWORK_NAME: "{{ entity_name }}" UNBOUND_RESOLVER_IP: "{{ lookup('config', application_id, 'networks.local.dns_resolver') }}" diff --git a/roles/svc-net-tor/templates/compose.yml.j2 b/roles/svc-net-tor/templates/compose.yml.j2 index 4284a342ca..59ddd84fdb 100644 --- a/roles/svc-net-tor/templates/compose.yml.j2 +++ b/roles/svc-net-tor/templates/compose.yml.j2 @@ -12,11 +12,9 @@ {{ lookup('compose_only', 'container_name', TOR_CONTAINER) }} {{ lookup('container_image', application_id, 'tor') }} {% if DEPLOYMENT_MODE == 'swarm' %} - ports: - - target: {{ TOR_SOCKS_PORT }} - published: {{ TOR_SOCKS_PORT }} - protocol: tcp - mode: host + {{ lookup('container_ports', + {'service': 'tor', 'protocol': 'socks', 'mode': 'host', + 'container': 'same'}) | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% else %} network_mode: "host" diff --git a/roles/svc-net-tor/vars/main.yml b/roles/svc-net-tor/vars/main.yml index 482668303f..6eb3bb1f87 100644 --- a/roles/svc-net-tor/vars/main.yml +++ b/roles/svc-net-tor/vars/main.yml @@ -6,7 +6,6 @@ TOR_IMAGE: "{{ lookup('config', application_id, 'services.tor.image') }}" TOR_VERSION: "{{ lookup('config', application_id, 'services.tor.version') }}" TOR_CONTAINER: "{{ lookup('config', application_id, 'services.tor.name') }}" -TOR_SOCKS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.socks') }}" TOR_NODE_ONION: "{{ lookup('config', application_id, 'services.tor.node', '') }}" TOR_USE_ENTRY_GUARDS: "{{ lookup('config', application_id, 'services.tor.use_entry_guards') | bool }}" TOR_FLAVOR: "{{ lookup('config', application_id, 'services.tor.flavor') }}" diff --git a/roles/svc-prx-openresty/meta/services.yml b/roles/svc-prx-openresty/meta/services.yml index d94903c97f..749d07cf9e 100644 --- a/roles/svc-prx-openresty/meta/services.yml +++ b/roles/svc-prx-openresty/meta/services.yml @@ -11,6 +11,17 @@ openresty: enabled: true shared: true name: openresty + ports: + internal: + http: 80 + https: 443 + public: + http: 80 + https: 443 + onion: + http: true + # nocheck: onion-flag the hidden service is already the transport encryption, and the onion address has no certificate a TLS listener could present + https: false healthcheck: test: ["CMD", "openresty", "-t", "-q"] min_storage: 264MB diff --git a/roles/svc-prx-openresty/templates/compose.yml.j2 b/roles/svc-prx-openresty/templates/compose.yml.j2 index 7d8f791608..b3bbe4721a 100644 --- a/roles/svc-prx-openresty/templates/compose.yml.j2 +++ b/roles/svc-prx-openresty/templates/compose.yml.j2 @@ -14,15 +14,9 @@ {{ lookup('compose_only', 'container_name', OPENRESTY_CONTAINER) }} {{ lookup('container_image', application_id, 'openresty') }} {% if DEPLOYMENT_MODE == 'swarm' %} - ports: - - target: 80 - published: 80 - protocol: tcp - mode: host - - target: 443 - published: 443 - protocol: tcp - mode: host + {{ lookup('container_ports', + {'service': 'openresty', 'protocol': 'http', 'mode': 'host'}, + {'service': 'openresty', 'protocol': 'https', 'mode': 'host'}) | indent(4) }} {{ lookup('container_extra_hosts', extra_hosts=['host.docker.internal:host-gateway']) | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% else %} diff --git a/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 b/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 index 58c4a51d5e..70439e95fa 100644 --- a/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 +++ b/roles/web-app-bigbluebutton/templates/compose.override.yml.j2 @@ -90,12 +90,12 @@ services: {% set service_name = 'coturn' %} {{ service_name }}: {{ lookup('container_volumes', service_name) | indent(4) }} - ports: - - "{{ BBB_TURN_PORT }}:{{ BBB_TURN_PORT }}/udp" - - "{{ BBB_TURN_PORT }}:{{ BBB_TURN_PORT }}/tcp" - - "{{ BBB_STUN_PORT }}:{{ BBB_STUN_PORT }}/udp" - - "{{ BBB_STUN_PORT }}:{{ BBB_STUN_PORT }}/tcp" - - "{{ BBB_RELAY_PORT_RANGE }}/udp" + {{ lookup('container_ports', + {'service': 'bigbluebutton', 'protocol': 'stun_turn', + 'transport': ['udp', 'tcp'], 'container': 'same'}, + {'service': 'bigbluebutton', 'protocol': 'relay', + 'transport': 'udp', 'container': 'same', + 'publish': 'ephemeral'}) | indent(4) }} command: >- --use-auth-secret --static-auth-secret=${TURN_SECRET} diff --git a/roles/web-app-bigbluebutton/vars/main.yml b/roles/web-app-bigbluebutton/vars/main.yml index 359128cfe8..b7eafd9df5 100644 --- a/roles/web-app-bigbluebutton/vars/main.yml +++ b/roles/web-app-bigbluebutton/vars/main.yml @@ -33,7 +33,6 @@ BBB_STUN_PORT: "{{ lookup('config', application_id, 'services.bigbluebutton.port 'services.coturn.ports.public.stun_turn_tls') }}" BBB_RELAY_PORT_START: "{{ lookup('config', application_id, 'services.bigbluebutton.ports.public.relay.start') }}" BBB_RELAY_PORT_END: "{{ lookup('config', application_id, 'services.bigbluebutton.ports.public.relay.end') }}" -BBB_RELAY_PORT_RANGE: "{{ BBB_RELAY_PORT_START }}-{{ BBB_RELAY_PORT_END }}" BBB_COLLABORA_ENABLED: "{{ not (lookup('config',application_id, 'services.collabora.shared')) }}" BBB_COLLABORA_PORT: "{{ lookup('config', 'web-svc-collabora', 'services.collabora.ports.internal.http') }}" diff --git a/roles/web-app-dashboard/templates/compose.yml.j2 b/roles/web-app-dashboard/templates/compose.yml.j2 index fe3ae4bc30..619e8c6420 100644 --- a/roles/web-app-dashboard/templates/compose.yml.j2 +++ b/roles/web-app-dashboard/templates/compose.yml.j2 @@ -1,12 +1,10 @@ {% include 'roles/sys-svc-compose/templates/base.yml.j2' %} {% set service_name = 'dashboard' %} - {% set container_port = lookup('config', application_id, 'services.' ~ service_name ~ '.ports.internal.http') %} {{ service_name }}: {% include 'roles/sys-svc-container/templates/deploy.yml.j2' %} {{ lookup('container_image', application_id, 'dashboard') }} {{ lookup('compose_only', 'container_name', DASHBOARD_CONTAINER) }} - ports: - - {{ DOCKER_BIND_HOST }}:{{ http_port }}:{{ container_port }} + {{ lookup('container_ports', ['dashboard', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('compose_restart') }} {{ lookup('container_networks') | indent(4) }} diff --git a/roles/web-app-discourse/templates/config.yml.j2 b/roles/web-app-discourse/templates/config.yml.j2 index 1312e2281d..f09ede8f90 100644 --- a/roles/web-app-discourse/templates/config.yml.j2 +++ b/roles/web-app-discourse/templates/config.yml.j2 @@ -15,8 +15,7 @@ templates: ## which TCP/IP ports should this container expose? ## If you want Discourse to share a port with another webserver like Apache or nginx, ## see https://meta.discourse.org/t/17247 for details -expose: - - "{{ DOCKER_BIND_HOST }}:{{ lookup('config', application_id, 'services.discourse.ports.local.http') }}:80" # http +{{ lookup('container_ports', ['discourse', 'http', DOCKER_BIND_HOST], key='expose') }} params: db_default_text_search_config: "pg_catalog.english" diff --git a/roles/web-app-funkwhale/templates/compose.yml.j2 b/roles/web-app-funkwhale/templates/compose.yml.j2 index 5d91f8e574..813ccd6228 100644 --- a/roles/web-app-funkwhale/templates/compose.yml.j2 +++ b/roles/web-app-funkwhale/templates/compose.yml.j2 @@ -30,8 +30,9 @@ {{ lookup('container_image', application_id, 'api') }} command: gunicorn {{ lookup('container_volumes', service_name) | indent(4) }} - ports: - - "{{ FUNKWHALE_DOCKER_API_PORT }}" + {{ lookup('container_ports', + {'service': 'api', 'protocol': 'http', + 'publish': 'ephemeral'}) | indent(4) }} {{ lookup('container_depends_on', application_id, extra={ 'migrate': {'condition': 'service_completed_successfully'}, }) | indent(4) }} diff --git a/roles/web-app-jitsi/templates/compose.yml.j2 b/roles/web-app-jitsi/templates/compose.yml.j2 index b893265663..d69c1d1fca 100644 --- a/roles/web-app-jitsi/templates/compose.yml.j2 +++ b/roles/web-app-jitsi/templates/compose.yml.j2 @@ -28,8 +28,9 @@ {{ lookup('compose_only', 'container_name', JITSI_JVB_CONTAINER) }} {% include 'roles/sys-svc-container/templates/base.yml.j2' %} {{ lookup('depends_on', {JITSI_PROSODY_SERVICE: 'service_started'}) }} - ports: - - "{{ JITSI_JVB_MEDIA_PORT }}:{{ JITSI_JVB_MEDIA_PORT }}/udp" # nocheck: compose-ports-must-use-container-ports + {{ lookup('container_ports', + {'service': 'jitsi', 'protocol': 'media', + 'transport': 'udp', 'container': 'same'}) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('container_networks') | indent(4) }} diff --git a/roles/web-app-keycloak/templates/sso_proxy/container.yml.j2 b/roles/web-app-keycloak/templates/sso_proxy/container.yml.j2 index 9ca1d59d3b..806fb3cd63 100644 --- a/roles/web-app-keycloak/templates/sso_proxy/container.yml.j2 +++ b/roles/web-app-keycloak/templates/sso_proxy/container.yml.j2 @@ -8,6 +8,7 @@ labels: infinito.sso-proxy.config-sha256: "{{ lookup('template', lookup('path_absolute', 'roles/web-app-keycloak/templates/sso_proxy/oauth2-proxy-keycloak.cfg.j2'), template_vars={'sso_proxy_application_id': application_id}) | hash('sha256') }}" ports: + # nocheck: compose-ports-must-use-container-ports the published port belongs to the consuming role's own entity and the container port to web-app-keycloak's sso_proxy, so the mapping spans two application_ids and no single container_ports call can read both - {{ DOCKER_BIND_HOST }}:{{ lookup('config', application_id, 'services.' ~ (application_id | get_entity_name) ~ '.ports.local.sso') }}:{{ lookup('config', 'web-app-keycloak', 'services.sso_proxy.ports.internal.http') }}/tcp volumes: - "{{ lookup('container', application_id, 'directories.volumes') }}{{ lookup('config', 'web-app-keycloak', 'services.sso_proxy.configuration_file') }}:/oauth2-proxy.cfg" diff --git a/roles/web-app-matrix/templates/flavor/ansible/services.yml.j2 b/roles/web-app-matrix/templates/flavor/ansible/services.yml.j2 index d0bb2d80ea..3746b3eb25 100644 --- a/roles/web-app-matrix/templates/flavor/ansible/services.yml.j2 +++ b/roles/web-app-matrix/templates/flavor/ansible/services.yml.j2 @@ -35,7 +35,8 @@ - /run - /run/lock - /tmp - ports: - - "{{ DOCKER_BIND_HOST }}:{{ MATRIX_SYNAPSE_PORT }}:8008" + {{ lookup('container_ports', ['synapse', 'http', DOCKER_BIND_HOST]) | indent(4) }} + # nocheck: compose-ports-must-use-container-ports matrix-docker-ansible-deploy fixes element on 8080 while this repo declares element.ports.internal.http 80; the two disagree and the playbook owns the container - "{{ DOCKER_BIND_HOST }}:{{ MATRIX_ELEMENT_PORT }}:8080" + # nocheck: compose-ports-must-use-container-ports the upstream playbook serves federation on 8048 and no ports.internal.federation is declared for it - "{{ DOCKER_BIND_HOST }}:{{ MATRIX_FEDERATION_PORT }}:8048" diff --git a/roles/web-app-matrix/templates/flavor/compose/services.yml.j2 b/roles/web-app-matrix/templates/flavor/compose/services.yml.j2 index 16e4954ca0..e24e06d6f3 100644 --- a/roles/web-app-matrix/templates/flavor/compose/services.yml.j2 +++ b/roles/web-app-matrix/templates/flavor/compose/services.yml.j2 @@ -1,7 +1,6 @@ {% set service_name = 'synapse' %} {{ service_name }}: {% include 'roles/sys-svc-container/templates/deploy.yml.j2' %} -{% set container_port = MATRIX_SYNAPSE_CONTAINER_PORT %} {{ lookup('container_image', application_id, 'synapse') }} {{ lookup('compose_only', 'container_name', MATRIX_SYNAPSE_NAME) }} {{ lookup('compose_restart') }} @@ -11,8 +10,7 @@ environment: - SYNAPSE_SERVER_NAME={{ MATRIX_SYNAPSE_DOMAIN }} - SYNAPSE_REPORT_STATS=no - ports: - - "{{ DOCKER_BIND_HOST }}:{{lookup('config', 'web-app-matrix', 'services.synapse.ports.local.http')}}:{{ container_port }}" + {{ lookup('container_ports', ['synapse', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_extra_hosts') | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} {{ lookup('container_depends_on', application_id) | indent(4) }} @@ -20,13 +18,11 @@ {% set service_name = 'element' %} {{ service_name }}: {% include 'roles/sys-svc-container/templates/deploy.yml.j2' %} -{% set container_port = lookup('config', application_id, 'services.' ~ service_name ~ '.ports.internal.http') %} {{ lookup('container_image', application_id, 'element') }} {{ lookup('compose_only', 'container_name', MATRIX_ELEMENT_NAME) }} {{ lookup('compose_restart') }} {{ lookup('container_volumes', service_name) | indent(4) }} - ports: - - "{{ DOCKER_BIND_HOST }}:{{ MATRIX_ELEMENT_PORT }}:{{ container_port }}" + {{ lookup('container_ports', ['element', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} {{ lookup('container_networks') | indent(4) }} diff --git a/roles/web-app-moodle/templates/compose.yml.j2 b/roles/web-app-moodle/templates/compose.yml.j2 index 5ce0b89800..0bb8966b57 100644 --- a/roles/web-app-moodle/templates/compose.yml.j2 +++ b/roles/web-app-moodle/templates/compose.yml.j2 @@ -40,8 +40,7 @@ {{ lookup('compose_only', 'container_name', MOODLE_NGINX_CONTAINER) }} {{ lookup('container_image', application_id, 'nginx') }} {{ lookup('depends_on', {'moodle': 'service_healthy'}) }} - ports: - - {{ DOCKER_BIND_HOST }}:{{ lookup('config', application_id, 'services.moodle.ports.local.http') }}:{{ MOODLE_NGINX_PORT }} + {{ lookup('container_ports', ['moodle', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('compose_restart') }} {{ lookup('container_networks') | indent(4) }} diff --git a/roles/web-app-nextcloud/templates/compose.yml.j2 b/roles/web-app-nextcloud/templates/compose.yml.j2 index cb6e1b68a1..c4a1edb7ee 100644 --- a/roles/web-app-nextcloud/templates/compose.yml.j2 +++ b/roles/web-app-nextcloud/templates/compose.yml.j2 @@ -75,11 +75,13 @@ init: true {% if NEXTCLOUD_HPB_NETWORK_MODE == 'bridge' %} {% if NEXTCLOUD_HPB_TURN_ONBOARD_ENABLED | bool and NEXTCLOUD_HPB_TURN_ONBOARD_PUBLISH_PORTS | bool %} - ports: - - {{ NEXTCLOUD_HPB_TURN_ONBOARD_BIND_HOST }}:{{ NEXTCLOUD_HPB_TURN_ONBOARD_PORT }}:{{ NEXTCLOUD_HPB_TURN_ONBOARD_PORT }}/tcp # nocheck: compose-ports-must-use-container-ports - - {{ NEXTCLOUD_HPB_TURN_ONBOARD_BIND_HOST }}:{{ NEXTCLOUD_HPB_TURN_ONBOARD_PORT }}:{{ NEXTCLOUD_HPB_TURN_ONBOARD_PORT }}/udp # nocheck: compose-ports-must-use-container-ports - - {{ NEXTCLOUD_HPB_TURN_ONBOARD_BIND_HOST }}:{{ NEXTCLOUD_HPB_TURN_ONBOARD_RELAY_PORT_START }}-{{ NEXTCLOUD_HPB_TURN_ONBOARD_RELAY_PORT_END }}:{{ NEXTCLOUD_HPB_TURN_ONBOARD_RELAY_PORT_START }}-{{ NEXTCLOUD_HPB_TURN_ONBOARD_RELAY_PORT_END }}/udp # nocheck: compose-ports-must-use-container-ports - - {{ NEXTCLOUD_HPB_TURN_ONBOARD_BIND_HOST }}:{{ NEXTCLOUD_HPB_TURN_ONBOARD_RELAY_PORT_START }}-{{ NEXTCLOUD_HPB_TURN_ONBOARD_RELAY_PORT_END }}:{{ NEXTCLOUD_HPB_TURN_ONBOARD_RELAY_PORT_START }}-{{ NEXTCLOUD_HPB_TURN_ONBOARD_RELAY_PORT_END }}/tcp # nocheck: compose-ports-must-use-container-ports + {{ lookup('container_ports', + {'service': 'nextcloud', 'protocol': 'stun_turn', + 'ip': NEXTCLOUD_HPB_TURN_ONBOARD_BIND_HOST, + 'transport': ['tcp', 'udp'], 'container': 'same'}, + {'service': 'nextcloud', 'protocol': 'relay', + 'ip': NEXTCLOUD_HPB_TURN_ONBOARD_BIND_HOST, + 'transport': ['udp', 'tcp'], 'container': 'same'}) | indent(4) }} {% endif %} expose: - "{{ container_port }}" diff --git a/roles/web-svc-coturn/templates/compose.yml.j2 b/roles/web-svc-coturn/templates/compose.yml.j2 index c54c7efdfd..e9b4b8dc58 100644 --- a/roles/web-svc-coturn/templates/compose.yml.j2 +++ b/roles/web-svc-coturn/templates/compose.yml.j2 @@ -18,13 +18,23 @@ {{ lookup('compose_only', 'container_name', COTURN_CONTAINER) }} network_mode: {{ COTURN_NETWORK_MODE }} {% if COTURN_NETWORK_MODE == 'bridge' %} - ports: - - "{{ COTURN_STUN_TURN_PORT }}:{{ COTURN_STUN_TURN_PORT }}/udp" # nocheck: compose-ports-must-use-container-ports - - "{{ COTURN_STUN_TURN_PORT }}:{{ COTURN_STUN_TURN_PORT }}/tcp" # nocheck: compose-ports-must-use-container-ports - {% if COTURN_TLS_ENABLED| bool %} - - "{{ COTURN_STUN_TURN_PORT_TLS }}:{{ COTURN_STUN_TURN_PORT_TLS }}/tcp" # nocheck: compose-ports-must-use-container-ports - {% endif %} - - "{{ COTURN_RELAY_PORT_RANGE }}/udp" # nocheck: compose-ports-must-use-container-ports + {% if COTURN_TLS_ENABLED | bool %} + {{ lookup('container_ports', + {'service': 'coturn', 'protocol': 'stun_turn', + 'transport': ['udp', 'tcp'], 'container': 'same'}, + {'service': 'coturn', 'protocol': 'stun_turn_tls', + 'transport': 'tcp', 'container': 'same'}, + {'service': 'coturn', 'protocol': 'relay', + 'transport': 'udp', 'container': 'same', + 'publish': 'ephemeral'}) | indent(4) }} + {% else %} + {{ lookup('container_ports', + {'service': 'coturn', 'protocol': 'stun_turn', + 'transport': ['udp', 'tcp'], 'container': 'same'}, + {'service': 'coturn', 'protocol': 'relay', + 'transport': 'udp', 'container': 'same', + 'publish': 'ephemeral'}) | indent(4) }} + {% endif %} {{ lookup('container_networks') | indent(4) }} {% endif %} {{ lookup('container_volumes', service_name) | indent(4) }} diff --git a/roles/web-svc-coturn/vars/main.yml b/roles/web-svc-coturn/vars/main.yml index c1bb58cc5f..1fd172c9aa 100644 --- a/roles/web-svc-coturn/vars/main.yml +++ b/roles/web-svc-coturn/vars/main.yml @@ -11,7 +11,6 @@ COTURN_STUN_TURN_PORT: "{{ lookup('config', application_id, 'services.coturn.por COTURN_STUN_TURN_PORT_TLS: "{{ lookup('config', application_id, 'services.coturn.ports.public.stun_turn_tls') }}" COTURN_RELAY_PORT_START: "{{ lookup('config', application_id, 'services.coturn.ports.public.relay.start') }}" COTURN_RELAY_PORT_END: "{{ lookup('config', application_id, 'services.coturn.ports.public.relay.end') }}" -COTURN_RELAY_PORT_RANGE: "{{ COTURN_RELAY_PORT_START }}-{{ COTURN_RELAY_PORT_END }}" COTURN_USER_NAME: "{{ lookup('users', 'coturnconsumer').username }}" COTURN_USER_PASSWORD: "{{ lookup('users', 'coturnconsumer').password }}" diff --git a/tests/lint/ansible/jinja/test_compose_ports_use_container_ports_lookup.py b/tests/lint/ansible/jinja/test_compose_ports_use_container_ports_lookup.py index 8e1758182f..5cb480b84e 100644 --- a/tests/lint/ansible/jinja/test_compose_ports_use_container_ports_lookup.py +++ b/tests/lint/ansible/jinja/test_compose_ports_use_container_ports_lookup.py @@ -7,8 +7,8 @@ declared ``ports.local`` / ``ports.public`` (host) and ``ports.internal`` (container) values, so the published and container ports stay one source of truth: - {{ lookup('container_ports', ['gitea', 'http'], ['gitea', 'ssh'], - ip=DOCKER_BIND_HOST) | indent(4) }} + {{ lookup('container_ports', ['gitea', 'http', DOCKER_BIND_HOST], + ['gitea', 'ssh']) | indent(4) }} Hand-writing the mapping ``- "{{ ... }}:{{ lookup('config', ..., 'services..ports.local.') }}:..."`` @@ -37,31 +37,94 @@ from . import PROJECT_ROOT _RULE = "compose-ports-must-use-container-ports" -_PORT_SEGMENT = re.compile(r"\A(?:\{\{.*\}\}|[\d.]+|\d+/(?:tcp|udp))\Z") +_PORT_SEGMENT = re.compile( + r"\A(?:\{\{.*\}\}|\$\{[^}]*\}|[\d.]+|[\d-]+)(?:/(?:tcp|udp))?\Z", +) -_LIST_ITEM = re.compile(r'-\s*"([^"]*)"') +_LIST_ITEM = re.compile(r"""-\s*["']?(.*?)["']?\s*(?:\#.*)?\Z""") + +_LONG_SYNTAX_KEY = re.compile(r"\A-?\s*(?:target|published|mode|protocol):") + +_BLOCK_KEY = re.compile(r"\A(ports|expose):\s*(.*)\Z") _PAIR = re.compile(r"\[\s*'([\w-]+)'\s*,\s*'([\w-]+)'\s*\]") def _is_scan_target(rel_path: str) -> bool: - if not rel_path.startswith("roles/") or "/templates/" not in rel_path: - return False - name = Path(rel_path).name - return name.endswith(".yml.j2") and "compose" in name + """Every role template that can carry a compose ``ports:`` block. + + ``compose.yml.j2`` and its ``compose..yml.j2`` siblings, plus the + ``*.yml.j2`` fragments they include: a mapping hand-written in an included + file publishes exactly the same port as one written inline. + """ + return ( + rel_path.startswith("roles/") + and "/templates/" in rel_path + and rel_path.endswith(".yml.j2") + ) -def _is_raw_port_mapping(line: str) -> bool: +def _is_raw_port_mapping(line: str, *, mapping_only: bool = False) -> bool: + """Whether *line* publishes a port by hand, quoted or not. + + ``mapping_only`` drops the single-segment form. Under ``expose:`` that form + names a container port for documentation and publishes nothing, so only a + ``host:container`` mapping there is a publication. + """ match = _LIST_ITEM.match(line.strip()) if match is None: return False - inner = match.group(1) - if "=" in inner: + inner = match.group(1).strip() + if not inner or "=" in inner: return False - segments = inner.split(":") - return len(segments) >= 2 and all( - _PORT_SEGMENT.match(segment.strip()) for segment in segments - ) + segments = [segment.strip() for segment in inner.split(":")] + if not all(_PORT_SEGMENT.match(segment) for segment in segments): + return False + if len(segments) >= 2: + return True + return not mapping_only and bool(_PORT_SEGMENT.match(inner)) + + +def _raw_port_lines(lines: list[str]) -> list[int]: + """1-based line numbers that publish a port by hand. + + The block is tracked by the indentation of its ``ports:`` / ``expose:`` key + rather than by the next key seen. Keying on the next line closed the block + on anything shaped like a key, which the long-syntax members ``target:``, + ``published:``, ``protocol:`` and ``mode:`` are, so the block could never + end while it was being read; blank, comment and ``{% %}`` lines equally have + no say in where a YAML block ends. + """ + found: list[int] = [] + block_indent: int | None = None + mapping_only = False + for number, line in enumerate(lines, start=1): + stripped = line.strip() + if not stripped or stripped.startswith(("#", "{%")): + continue + indent = len(line) - len(line.rstrip("\n").lstrip()) + if ( + block_indent is not None + and indent <= block_indent + and not stripped.startswith("- ") + ): + block_indent = None + if block_indent is None: + opener = _BLOCK_KEY.match(stripped) + if opener: + block_indent = indent + mapping_only = opener.group(1) == "expose" + inline = opener.group(2).strip().strip("[]") + if inline and _is_raw_port_mapping( + "- " + inline, mapping_only=mapping_only + ): + found.append(number) + continue + if _LONG_SYNTAX_KEY.match(stripped) or _is_raw_port_mapping( + line, mapping_only=mapping_only + ): + found.append(number) + return found class TestComposePortsUseContainerPortsLookup(unittest.TestCase): @@ -75,27 +138,9 @@ class TestComposePortsUseContainerPortsLookup(unittest.TestCase): if not _is_scan_target(rel): continue lines = content.splitlines() - in_ports_block = False - for idx, line in enumerate(lines): - stripped = line.strip() - if not stripped: - continue - if stripped.startswith("- "): - if ( - in_ports_block - and _is_raw_port_mapping(line) - and not is_suppressed_at( - lines, idx + 1, _RULE, mode="same-or-above" - ) - ): - findings.append((rel, idx + 1, stripped)) - continue - if stripped.startswith("{{"): - in_ports_block = "container_ports" in stripped - continue - key = re.match(r"[\w-]+:", stripped) - if key: - in_ports_block = key.group(0) == "ports:" + for number in _raw_port_lines(lines): + if not is_suppressed_at(lines, number, _RULE, mode="same-or-above"): + findings.append((rel, number, lines[number - 1].strip())) if findings: formatted = "\n".join( @@ -107,8 +152,8 @@ class TestComposePortsUseContainerPortsLookup(unittest.TestCase): "Build the `ports:` block with the container_ports lookup instead, " "so the published (local/public) and container (internal) ports stay " "a single declared source of truth:\n\n" - " {{ lookup('container_ports', ['', ''],\n" - " ip=DOCKER_BIND_HOST) | indent(4) }}\n\n" + " {{ lookup('container_ports',\n" + " ['', '', DOCKER_BIND_HOST]) | indent(4) }}\n\n" "One ['', ''] pair per published port; the lookup reads " "ports.local/public for the host side and ports.internal for the " "container side. Mark with " diff --git a/tests/unit/python/plugins/lookup/test_container_ports.py b/tests/unit/python/plugins/lookup/test_container_ports.py index cf854230f9..385946855a 100644 --- a/tests/unit/python/plugins/lookup/test_container_ports.py +++ b/tests/unit/python/plugins/lookup/test_container_ports.py @@ -119,5 +119,133 @@ class TestContainerPortsLookup(unittest.TestCase): self.assertEqual(out, ['ports:\n - "10.0.0.1:8002:3000"']) +def _edge_apps(): + return { + "web-svc-coturn": { + "services": { + "coturn": { + "ports": { + "public": { + "stun_turn": 3481, + "relay": {"start": 20000, "end": 39999}, + } + } + }, + "openresty": { + "ports": { + "internal": {"http": 80}, + "public": {"http": 80}, + } + }, + "api": {"ports": {"internal": {"http": 5000}}}, + } + } + } + + +def _run_edge(terms): + lookup = LookupModule() + lookup._loader = mock.MagicMock() + apps = _edge_apps() + with patch("plugins.lookup.container_ports.lookup_loader") as loader_mock: + loader_mock.get.return_value = mock.MagicMock(run=lambda *_a, **_k: [apps]) + return lookup.run(terms, variables={"application_id": "web-svc-coturn"})[0] + + +class TestContainerPortsDictTerms(unittest.TestCase): + def test_one_list_item_per_transport(self): + out = _run_edge( + [ + { + "service": "coturn", + "protocol": "stun_turn", + "transport": ["udp", "tcp"], + "container": "same", + } + ] + ) + self.assertEqual(out, 'ports:\n - "3481:3481/udp"\n - "3481:3481/tcp"') + + def test_a_range_renders_on_both_sides(self): + out = _run_edge( + [ + { + "service": "coturn", + "protocol": "relay", + "transport": "udp", + "container": "same", + } + ] + ) + self.assertEqual(out, 'ports:\n - "20000-39999:20000-39999/udp"') + + def test_host_mode_emits_the_long_block(self): + out = _run_edge([{"service": "openresty", "protocol": "http", "mode": "host"}]) + self.assertEqual( + out, + "ports:\n - target: 80\n published: 80\n" + " protocol: tcp\n mode: host", + ) + + def test_ephemeral_publishes_only_the_container_port(self): + out = _run_edge( + [{"service": "api", "protocol": "http", "publish": "ephemeral"}] + ) + self.assertEqual(out, 'ports:\n - "5000"') + + def test_host_mode_rejects_an_ip(self): + with self.assertRaises(AnsibleError): + _run_edge( + [ + { + "service": "openresty", + "protocol": "http", + "mode": "host", + "ip": "10.0.0.1", + } + ] + ) + + def test_an_unknown_term_key_is_rejected(self): + with self.assertRaises(AnsibleError): + _run_edge([{"service": "openresty", "protocol": "http", "protokol": "tcp"}]) + + def test_key_names_the_block(self): + lookup = LookupModule() + lookup._loader = mock.MagicMock() + with patch("plugins.lookup.container_ports.lookup_loader") as loader_mock: + loader_mock.get.return_value = mock.MagicMock( + run=lambda *_a, **_k: [_edge_apps()] + ) + out = lookup.run( + [["openresty", "http"]], + variables={"application_id": "web-svc-coturn"}, + key="expose", + ) + self.assertEqual(out, ['expose:\n - "80:80"']) + + def test_an_unknown_block_key_is_rejected(self): + lookup = LookupModule() + lookup._loader = mock.MagicMock() + with ( + patch("plugins.lookup.container_ports.lookup_loader") as loader_mock, + self.assertRaises(AnsibleError), + ): + loader_mock.get.return_value = mock.MagicMock( + run=lambda *_a, **_k: [_edge_apps()] + ) + lookup.run( + [["openresty", "http"]], + variables={"application_id": "web-svc-coturn"}, + key="publish", + ) + + def test_an_unknown_transport_is_rejected(self): + with self.assertRaises(AnsibleError): + _run_edge( + [{"service": "openresty", "protocol": "http", "transport": "sctp"}] + ) + + if __name__ == "__main__": unittest.main() From b71cf359b4942ba57125f2751305be6c8efda17d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 16:59:54 +0200 Subject: [PATCH 039/197] feat(networks): declare the deployment's own address space once The private blocks were written out wherever something needed to say "this is us": NETWORK_TRUSTED_PROXY_CIDRS listed all three, Tor's SocksPolicy repeated them, unbound's access-control repeated them again. Four copies of a fact, and nothing tying them together, so narrowing one narrows nothing else. Worse, the block role networks are carved from was never declared at all. The allocator behind 'infinito contributing network address suggest' derives its umbrella from the subnets already in use, so the tree defined the rule instead of the rule defining the tree. Anything consuming "our own address space" had to guess it, and a guess that is one role out of date silently excludes that role. Measured: 106 of 107 role subnets sit in 192.168.0.0/16 and one does not, so a firewall range derived from the pool by eye would have cut BigBlueButton off. The blocks are now named once and everything else is derived, NETWORK_TRUSTED_PROXY_CIDRS included, which renders byte-identically to before for its four consumers. BigBlueButton stays outside because its upstream pins both the block and its gateway, which meta/services.yml repeats as upstream_host 10.7.7.1. The exception list therefore carries the role id rather than the CIDR: the block is already declared in that role's meta/networks.yml, and repeating it here would be the same duplication one layer up. The role_subnets lookup resolves the ids and raises on a role that declares no subnet, because a silently dropped entry would narrow whatever firewall or trust list consumes the result. A lint keeps the declaration honest in both directions: every role subnet must sit in the declared space, and an exception whose block the pool already admits is rejected so it cannot outlive its reason. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/08_networks.yml | 17 ++-- plugins/lookup/role_subnets.py | 64 ++++++++++++ .../roles/meta/test_role_subnet_pool.py | 97 +++++++++++++++++++ 3 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 plugins/lookup/role_subnets.py create mode 100644 tests/lint/ansible/roles/meta/test_role_subnet_pool.py diff --git a/group_vars/all/08_networks.yml b/group_vars/all/08_networks.yml index 25f3a66890..c90530f319 100644 --- a/group_vars/all/08_networks.yml +++ b/group_vars/all/08_networks.yml @@ -6,12 +6,17 @@ NETWORK_PUBLIC_DNS_RESOLVERS: - 1.0.0.1 # nocheck: hardcoded-dns-resolver NETWORK_DOCKER_EMBEDDED_DNS_RESOLVER: "127.0.0.11" # nocheck: hardcoded-dns-resolver NETWORK_DOCKER_DNS_NGINX_DIRECTIVE: "resolver {{ NETWORK_DOCKER_EMBEDDED_DNS_RESOLVER }} valid=30s ipv6={{ NETWORK_IPV6_ENABLED | on_off }};" -NETWORK_TRUSTED_PROXY_CIDRS: - - 127.0.0.1 - - "::1" - - 10.0.0.0/8 - - 172.16.0.0/12 - - 192.168.0.0/16 +NETWORK_LOOPBACK_CIDR: "127.0.0.0/8" +NETWORK_PRIVATE_CIDRS: + class_a: "10.0.0.0/8" + class_b: "172.16.0.0/12" + class_c: "192.168.0.0/16" +NETWORK_INTERNAL_CIDRS: "{{ [NETWORK_LOOPBACK_CIDR] + (NETWORK_PRIVATE_CIDRS.values() | list) }}" +NETWORK_ROLE_SUBNET_POOL: "{{ NETWORK_PRIVATE_CIDRS.class_c }}" +NETWORK_ROLE_SUBNET_EXCEPTION_ROLES: + - web-app-bigbluebutton +NETWORK_SWARM_GWBRIDGE_POOL: "{{ NETWORK_PRIVATE_CIDRS.class_b }}" +NETWORK_TRUSTED_PROXY_CIDRS: "{{ ['127.0.0.1', '::1'] + (NETWORK_PRIVATE_CIDRS.values() | list) }}" NETWORK_DOCKER_ADDRESS_POOLS: - base: "10.208.0.0/12" size: 24 diff --git a/plugins/lookup/role_subnets.py b/plugins/lookup/role_subnets.py new file mode 100644 index 0000000000..b3a06976a1 --- /dev/null +++ b/plugins/lookup/role_subnets.py @@ -0,0 +1,64 @@ +"""Lookup `role_subnets`: the local subnets other roles declare. + +Single SPOT for "which address block does role X own". The block is declared +once, in that role's ``meta/networks.yml``; anything that has to reason about +the deployment's own address space reads it from here instead of repeating the +literal, which is how a CIDR ends up stated twice and then drifts. + +Usage: + + {{ lookup('role_subnets', 'web-app-bigbluebutton') }} # ['10.7.7.0/24'] + {{ lookup('role_subnets', SOME_ROLE_LIST) }} # one entry per role + +Terms may be role ids or lists of them, mixed. A role that declares no local +subnet raises, because a silently dropped entry would narrow whatever firewall +or trust list consumes the result. +""" + +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleError +from ansible.plugins.loader import lookup_loader +from ansible.plugins.lookup import LookupBase + +_SUBNET_PATH = "networks.local.subnet" + + +def _flattened(terms: list[Any]) -> list[str]: + roles: list[str] = [] + for term in terms: + if isinstance(term, (list, tuple)): + roles.extend(str(entry).strip() for entry in term) + else: + roles.append(str(term).strip()) + return [role for role in roles if role] + + +class LookupModule(LookupBase): + def run( + self, + terms: list[Any] | None, + variables: dict[str, Any] | None = None, + **kwargs: Any, + ) -> list[Any]: + roles = _flattened(terms or []) + if not roles: + raise AnsibleError( + "lookup('role_subnets', ) expects " + "at least one role id" + ) + config = lookup_loader.get( + "config", + loader=getattr(self, "_loader", None), + templar=getattr(self, "_templar", None), + ) + subnets: list[str] = [] + for role in roles: + value = config.run([role, _SUBNET_PATH], variables=variables)[0] + text = "" if value is None else str(value).strip() + if not text: + raise AnsibleError(f"role_subnets: '{role}' declares no {_SUBNET_PATH}") + subnets.append(text) + return [subnets] diff --git a/tests/lint/ansible/roles/meta/test_role_subnet_pool.py b/tests/lint/ansible/roles/meta/test_role_subnet_pool.py new file mode 100644 index 0000000000..b18f584947 --- /dev/null +++ b/tests/lint/ansible/roles/meta/test_role_subnet_pool.py @@ -0,0 +1,97 @@ +"""Lint: a role's local subnet comes out of the declared pool. + +``NETWORK_ROLE_SUBNET_POOL`` in ``group_vars/all/08_networks.yml`` names the +block role networks are carved from. Until it existed the block was an emergent +property: the allocator in ``cli contributing network address suggest`` derives +its umbrella from the subnets already in use, so the tree defined the rule +instead of the rule defining the tree. Anything that consumes "our own address +space" then has to guess, and a guess that is one role out of date silently +excludes that role. That is not hypothetical: a firewall range derived from the +pool by eye would have cut BigBlueButton off, because its subnet is the one +that sits outside. + +A role that genuinely cannot choose its block is named in +``NETWORK_ROLE_SUBNET_EXCEPTION_ROLES``. The list carries role ids rather than +CIDRs on purpose: the block is already declared in that role's +``meta/networks.yml``, and repeating it here would be the same duplication one +layer up. Consumers resolve the ids through ``lookup('role_subnets', ...)``. +""" + +from __future__ import annotations + +import ipaddress +import unittest + +import jinja2 + +from utils.cache.yaml import load_yaml_any +from utils.meta.scan import iter_subnets + +from . import PROJECT_ROOT + +NETWORKS_FILE = PROJECT_ROOT / "group_vars" / "all" / "08_networks.yml" + + +def _declared_space() -> tuple[list[ipaddress.IPv4Network], list[str]]: + """The pool plus the blocks of the excepted roles, with their raw strings.""" + data = load_yaml_any(str(NETWORKS_FILE), default_if_missing={}) or {} + pool = data.get("NETWORK_ROLE_SUBNET_POOL") + if isinstance(pool, str) and "{{" in pool: + pool = jinja2.Template(pool).render(**data) + roles = data.get("NETWORK_ROLE_SUBNET_EXCEPTION_ROLES") or [] + if not isinstance(pool, str) or not isinstance(roles, list): + raise TypeError( + f"{NETWORKS_FILE} must declare NETWORK_ROLE_SUBNET_POOL as a string " + "and NETWORK_ROLE_SUBNET_EXCEPTION_ROLES as a list of role ids" + ) + declared = dict(iter_subnets()) + raw = [pool] + for role in roles: + subnet = declared.get(str(role)) + if subnet is None: + raise AssertionError( + f"{NETWORKS_FILE}: NETWORK_ROLE_SUBNET_EXCEPTION_ROLES names " + f"'{role}', which declares no networks.local.subnet" + ) + raw.append(str(subnet)) + return [ipaddress.ip_network(entry, strict=True) for entry in raw], raw + + +class TestRoleSubnetPool(unittest.TestCase): + def test_every_role_subnet_sits_in_the_declared_space(self) -> None: + allowed, raw = _declared_space() + findings = [ + f"- {role}: {subnet}" + for role, subnet in sorted(iter_subnets(), key=lambda item: item[0]) + if not any(subnet.subnet_of(block) for block in allowed) + ] + if findings: + self.fail( + "These roles declare networks.local.subnet outside the address " + "space this deployment declares for itself:\n" + + "\n".join(findings) + + "\n\nDeclared space: " + + ", ".join(raw) + + "\n\nFix: pick a subnet inside the pool via " + "`infinito contributing network address suggest`, or, when the " + "block is pinned by something outside this repository, name the " + "role in NETWORK_ROLE_SUBNET_EXCEPTION_ROLES in " + "group_vars/all/08_networks.yml." + ) + + def test_the_exceptions_are_not_already_covered(self) -> None: + """An exception inside the pool is dead weight that outlives its reason.""" + allowed, raw = _declared_space() + pool = allowed[0] + redundant = [entry for entry in allowed[1:] if entry.subnet_of(pool)] + if redundant: + self.fail( + "NETWORK_ROLE_SUBNET_EXCEPTION_ROLES names role(s) whose subnet " + f"already sits inside {raw[0]}: " + f"{', '.join(str(entry) for entry in redundant)}. " + "Remove them; the pool already admits those addresses." + ) + + +if __name__ == "__main__": + unittest.main() From 46491b1d3a7c67bf2213b6eb34045090f0a80e87 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 17:01:27 +0200 Subject: [PATCH 040/197] test(lint): check the shell scripts that ship as jinja templates make lint-shellcheck walks files on disk, so a .sh.j2 is invisible to it. The thing that actually runs on the host is the render, and nothing looked at the render until it failed on a node. That is an unpleasant place to discover a quoting mistake in a script which runs as root and rewrites the host packet filter. Every roles/*/templates/*.sh.j2 is now rendered and handed to bash -n, plus shellcheck where it is installed. A value the check does not know renders as a placeholder token, an unknown collection renders as empty, an unknown filter passes its value through, and lookup returns a token, so a template written for ansible still renders to whole bash and the check judges the shape of the script rather than the values one deployment happens to supply. SC2034 is excluded with its reason: emptying a conditional branch orphans the variables that branch used, so a partial render cannot tell a genuinely unused variable from one whose only reader was not rendered. Everything else stays on. First run, first finding: the backup ssh wrapper substituted with backticks. Co-Authored-By: Claude Opus 5 (1M context) --- roles/user-backup/templates/ssh-wrapper.sh.j2 | 2 +- .../jinja/test_shell_templates_are_valid.py | 153 ++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 tests/lint/ansible/jinja/test_shell_templates_are_valid.py diff --git a/roles/user-backup/templates/ssh-wrapper.sh.j2 b/roles/user-backup/templates/ssh-wrapper.sh.j2 index fdffae7363..8ef003fc56 100644 --- a/roles/user-backup/templates/ssh-wrapper.sh.j2 +++ b/roles/user-backup/templates/ssh-wrapper.sh.j2 @@ -3,7 +3,7 @@ # log command if [ -n "$SSH_ORIGINAL_COMMAND" ] then - echo "`/bin/date`: $SSH_ORIGINAL_COMMAND" | systemd-cat -t "ssh-wrapper.sh" + echo "$(/bin/date): $SSH_ORIGINAL_COMMAND" | systemd-cat -t "ssh-wrapper.sh" fi get_hashed_machine_id="sha256sum /etc/machine-id"; diff --git a/tests/lint/ansible/jinja/test_shell_templates_are_valid.py b/tests/lint/ansible/jinja/test_shell_templates_are_valid.py new file mode 100644 index 0000000000..7df56911dc --- /dev/null +++ b/tests/lint/ansible/jinja/test_shell_templates_are_valid.py @@ -0,0 +1,153 @@ +"""Lint: a shell script shipped as a Jinja template still gets checked. + +``make lint-shellcheck`` walks files on disk, so a ``*.sh.j2`` is invisible to +it: the thing that actually runs on the host is the render, and nothing looks at +the render until it fails on a node. That is an unpleasant place to discover a +quoting mistake in a script which runs as root and rewrites the host packet +filter. + +Every ``roles/*/templates/*.sh.j2`` is rendered here and handed to ``bash -n``, +plus ``shellcheck`` where it is installed. A value the lint does not know +renders as a placeholder token and an unknown collection renders as empty, so +the check judges the shape of the script rather than what a particular +deployment puts in it. Where a template's structure only becomes real with +concrete values, those live in ``_SAMPLE``. + +``SC2034`` is excluded: emptying a conditional branch orphans the variables that +branch used, so a partial render cannot tell a genuinely unused variable from +one whose only reader was not rendered. +""" + +from __future__ import annotations + +import shlex +import shutil +import subprocess +import unittest +from pathlib import Path + +import jinja2 + +from utils.cache.files import iter_project_files, read_text + +from . import PROJECT_ROOT + +_PARTIAL_RENDER_NOISE = "SC2034" + + +class _PassthroughFilters(dict): + """Any filter this environment does not know leaves the value untouched. + + The repo's own filter plugins live in ansible, not in bare jinja2, and the + check is about the shape of the resulting shell rather than about what a + filter computes. + """ + + def __missing__(self, name: str): + def _identity(value: object, *_args: object, **_kwargs: object) -> object: + return value + + return _identity + + def __contains__(self, name: object) -> bool: + """Claim every filter, because jinja resolves them at compile time.""" + return True + + def get(self, name: object, default: object = None) -> object: + """Jinja looks filters up with ``get``, which skips ``__missing__``.""" + return self[name] if name in dict.keys(self) else self.__missing__(str(name)) + + +class _Placeholder(jinja2.Undefined): + """An unknown value that keeps the rendered script syntactically whole.""" + + def __str__(self) -> str: + return "placeholder" + + def __iter__(self): + return iter(()) + + def __bool__(self) -> bool: + return False + + def __getattr__(self, name: str) -> _Placeholder: + return self + + def __getitem__(self, key: object) -> _Placeholder: + return self + + +_SAMPLE = { + "TOR_EGRESS_GUARD_CHAIN": "INFINITO_TOR_EGRESS", + "TOR_EGRESS_CLIENT_CIDRS": ["127.0.0.0/8", "10.208.0.0/12"], + "TOR_EGRESS_GUARD_HOOK_CHAINS": ["INPUT", "DOCKER-USER"], + "TOR_EGRESS_GUARDED_PORTS": [ + {"port": "9040", "protocol": "tcp"}, + {"port": "9053", "protocol": "udp"}, + ], +} + + +def _shell_templates() -> list[Path]: + found = [] + for path_str in iter_project_files(extensions=(".j2",)): + rel = Path(path_str).relative_to(PROJECT_ROOT).as_posix() + if rel.startswith("roles/") and rel.endswith(".sh.j2"): + found.append(Path(path_str)) + return sorted(found) + + +class TestShellTemplatesAreValid(unittest.TestCase): + def test_every_shell_template_renders_to_valid_bash(self) -> None: + templates = _shell_templates() + self.assertTrue(templates, "no roles/*/templates/*.sh.j2 found to check") + shellcheck = shutil.which("shellcheck") + for path in templates: + rel = path.relative_to(PROJECT_ROOT).as_posix() + with self.subTest(template=rel): + environment = jinja2.Environment( + undefined=_Placeholder, + keep_trailing_newline=True, + autoescape=False, # noqa: S701 shell, not markup + ) + environment.filters = _PassthroughFilters(environment.filters) + environment.filters["quote"] = shlex.quote + environment.globals["lookup"] = lambda *_args, **_kwargs: "placeholder" + rendered = environment.from_string(read_text(str(path))).render( + **_SAMPLE + ) + syntax = subprocess.run( + ["bash", "-n"], + input=rendered, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual( + syntax.returncode, + 0, + f"{rel} does not render to valid bash:\n{syntax.stderr}", + ) + if shellcheck is None: + continue + report = subprocess.run( + [ + shellcheck, + "--shell=bash", + f"--exclude={_PARTIAL_RENDER_NOISE}", + "-", + ], + input=rendered, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual( + report.returncode, + 0, + f"{rel} fails shellcheck once rendered:\n{report.stdout}", + ) + + +if __name__ == "__main__": + unittest.main() From b804060e543e8b8c7784c68e3f02076fcd7750dc Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 17:03:02 +0200 Subject: [PATCH 041/197] fix(lint): let the two port lints see the edge proxy as the edge b036f6003 gave svc-prx-openresty the ports it had been binding as literals, including ports.public.http and the onion flag that turns the node onion's port 80 into a declaration instead of a hardcoded torrc line. The matching lint change stayed in the working tree, so that commit does not pass its own gate: the onion-flag lint excludes http wholesale, never finds the category among the declared ones, and reports the flag as out of scope. The exclusion was right for the 128 roles that declare http, sso or websocket on the loopback and are reached through the proxy on the onion's port 80. It was wrong for the single role that publishes http on the host, because that role is the proxy. The lint now splits on scope rather than on category: exempt under local, decision required under public. The companion lint gets the same treatment it needed for the dict-form call sites: the ports block is tracked by the indentation of its key rather than by the next line that looks like one, since target, published, protocol and mode are all valid keys and closed the block while it was still being read. Under expose: only a real host:container mapping counts, because there the single-segment form documents a container port and publishes nothing. Co-Authored-By: Claude Opus 5 (1M context) --- ...test_compose_ports_use_container_ports_lookup.py | 8 +++++--- tests/lint/ansible/services/test_onion_port_flag.py | 13 +++++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/lint/ansible/jinja/test_compose_ports_use_container_ports_lookup.py b/tests/lint/ansible/jinja/test_compose_ports_use_container_ports_lookup.py index 5cb480b84e..d66c3464c9 100644 --- a/tests/lint/ansible/jinja/test_compose_ports_use_container_ports_lookup.py +++ b/tests/lint/ansible/jinja/test_compose_ports_use_container_ports_lookup.py @@ -138,9 +138,11 @@ class TestComposePortsUseContainerPortsLookup(unittest.TestCase): if not _is_scan_target(rel): continue lines = content.splitlines() - for number in _raw_port_lines(lines): - if not is_suppressed_at(lines, number, _RULE, mode="same-or-above"): - findings.append((rel, number, lines[number - 1].strip())) + findings.extend( + (rel, number, lines[number - 1].strip()) + for number in _raw_port_lines(lines) + if not is_suppressed_at(lines, number, _RULE, mode="same-or-above") + ) if findings: formatted = "\n".join( diff --git a/tests/lint/ansible/services/test_onion_port_flag.py b/tests/lint/ansible/services/test_onion_port_flag.py index 6e5fc32ab5..04b839edc6 100644 --- a/tests/lint/ansible/services/test_onion_port_flag.py +++ b/tests/lint/ansible/services/test_onion_port_flag.py @@ -74,18 +74,27 @@ _MARKER_RE = re.compile( def _host_bound_categories(ports: Mapping) -> set[str]: - """Category names of *ports* that bind a host address as a single TCP port.""" + """Category names of *ports* that bind a host address as a single TCP port. + + The proxy-fronted categories are skipped under ``local`` and kept under + ``public``, which is the difference between riding behind the edge proxy and + being it: 128 roles declare ``http``/``sso``/``websocket`` on the loopback, + and exactly one publishes ``http`` on the host. + """ found: set[str] = set() for scope in _HOST_BOUND_SCOPES: declared = ports.get(scope) if not isinstance(declared, Mapping): continue + skipped = UDP_ONLY_CATEGORIES | _IMPLICIT_TLS_CATEGORIES + if scope == "local": + skipped = skipped | _PROXY_FRONTED_CATEGORIES found.update( category for category, value in declared.items() if isinstance(value, int) and not isinstance(value, bool) - and category not in _OUT_OF_SCOPE + and category not in skipped ) return found From fdaa31d6cb03513500342c8452b2810c3badc0b2 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 17:13:12 +0200 Subject: [PATCH 042/197] feat(svc-net-tor): carry mail to a .onion in both deploy modes Sending to a .onion needs two things of the node: a resolver that answers the suffix, and a route from the container to Tor's transparent port. Neither existed. Compose pointed Mailu at unbound, which knows nothing about .onion, and in swarm Tor's DNSPort and TransPort listen inside the container where the host's dnsmasq and nat rules cannot reach them, so the forwarding rules that were already there pointed at a closed port on every swarm node. Compose resolves through unbound, which now forwards the onion zone to Tor's DNSPort on the bridge it already owns. The forward alone would have answered SERVFAIL: the signed root proves that onion does not exist, so with auto-trust-anchor-file set every answer below it validates as bogus, and Tor's DNSPort can supply no DS. The zone is therefore declared insecure. Swarm resolves through the node dnsmasq instead, which every container already uses because container_dns puts the bridge resolver into daemon.json wherever svc-net-tor is deployed, and which now reaches Tor because its ports are published host-mode next to the SOCKS port that was published there already. The transport rule changed target rather than mechanism. REDIRECT rewrites the destination to the primary address of the interface the packet arrived on, which is a different project bridge for every role, so Tor would have had to listen on all of them. A DNAT naming the bridge keeps one listener, and in swarm the same address reaches the published port through the proxy docker runs there. That leaves the exposure. SocksPort has SocksPolicy; DNSPort and TransPort have no equivalent, and a swarm-published port carries no host address to bind to, so in swarm the packet filter is their only boundary. Compose needs none, because there Tor binds loopback and the bridge and nothing else. The swarm guard admits this deployment's own address space and drops the rest, and it is installed as a unit ordered after docker rather than as ansible-applied rules: nothing here persists netfilter state, so docker recreating its chains at boot would have left Tor back on the wildcard with the guard gone. It rebuilds its chain instead of appending to it, because the appending version could only ever be loosened. It hooks INPUT and DOCKER-USER both, since host-local traffic reaches a published port through docker's proxy and external traffic through FORWARD. The deploy now proves all of it back and fails when a rule is missing, the router no longer runs on swarm nodes that carry no Tor, and the CLI test probes the forwarded ports over SOCKS instead of assuming a HiddenServicePort line means a reachable service. Measured so far: Tor accepts the non-loopback binds and reports circuit-established, and the chain, its order and the DOCKER-USER hooks read back correctly from a deployed host. The swarm path is not yet exercised. Co-Authored-By: Claude Opus 5 (1M context) --- .../svc-dns-unbound/templates/unbound.conf.j2 | 16 +- roles/svc-dns-unbound/vars/main.yml | 4 + roles/svc-net-tor/files/test/onion_ports.py | 166 ++++++++++++++++++ roles/svc-net-tor/files/test/test.sh | 3 + roles/svc-net-tor/handlers/main.yml | 8 + .../lookup_plugins/tor_egress_clients.py | 95 ++++++++++ roles/svc-net-tor/tasks/00_core.yml | 4 +- roles/svc-net-tor/tasks/router.yml | 89 ++++++++++ roles/svc-net-tor/templates/compose.yml.j2 | 6 +- roles/svc-net-tor/templates/test.env.j2 | 4 + .../templates/tor-egress-guard.service.j2 | 13 ++ .../templates/tor-egress-guard.sh.j2 | 21 +++ roles/svc-net-tor/templates/torrc.j2 | 17 +- roles/svc-net-tor/vars/main.yml | 15 ++ .../svc-net-tor/test_egress_guard_script.py | 110 ++++++++++++ .../svc-net-tor/test_onion_ports_probe.py | 162 +++++++++++++++++ 16 files changed, 722 insertions(+), 11 deletions(-) create mode 100755 roles/svc-net-tor/files/test/onion_ports.py create mode 100644 roles/svc-net-tor/handlers/main.yml create mode 100644 roles/svc-net-tor/lookup_plugins/tor_egress_clients.py create mode 100644 roles/svc-net-tor/templates/tor-egress-guard.service.j2 create mode 100644 roles/svc-net-tor/templates/tor-egress-guard.sh.j2 create mode 100644 tests/unit/python/roles/svc-net-tor/test_egress_guard_script.py create mode 100644 tests/unit/python/roles/svc-net-tor/test_onion_ports_probe.py diff --git a/roles/svc-dns-unbound/templates/unbound.conf.j2 b/roles/svc-dns-unbound/templates/unbound.conf.j2 index d9f07c33b3..047486d8da 100644 --- a/roles/svc-dns-unbound/templates/unbound.conf.j2 +++ b/roles/svc-dns-unbound/templates/unbound.conf.j2 @@ -7,10 +7,9 @@ server: do-udp: yes do-tcp: yes do-daemonize: no - access-control: 127.0.0.0/8 allow - access-control: 10.0.0.0/8 allow - access-control: 172.16.0.0/12 allow - access-control: 192.168.0.0/16 allow +{% for _cidr in NETWORK_INTERNAL_CIDRS %} + access-control: {{ _cidr }} allow +{% endfor %} directory: "{{ lookup('volume', 'svc-dns-unbound', 'data').target }}" username: unbound auto-trust-anchor-file: trusted-key.key @@ -18,6 +17,9 @@ server: hide-identity: yes hide-version: yes cache-min-ttl: 300 +{% if UNBOUND_ONION_FORWARD_ADDR | length > 0 %} + domain-insecure: "onion." +{% endif %} remote-control: control-enable: yes @@ -30,3 +32,9 @@ forward-zone: forward-addr: {{ resolver }} {% endfor %} {% endif %} +{% if UNBOUND_ONION_FORWARD_ADDR | length > 0 %} + +forward-zone: + name: "onion." + forward-addr: {{ UNBOUND_ONION_FORWARD_ADDR }} +{% endif %} diff --git a/roles/svc-dns-unbound/vars/main.yml b/roles/svc-dns-unbound/vars/main.yml index 065d64f95f..75207698c0 100644 --- a/roles/svc-dns-unbound/vars/main.yml +++ b/roles/svc-dns-unbound/vars/main.yml @@ -11,3 +11,7 @@ UNBOUND_VOLUMES_DIR: "{{ lookup('container', application_id, 'directories.volume UNBOUND_CONF_FILE: "{{ [ UNBOUND_VOLUMES_DIR, 'unbound.conf' ] | path_join }}" UNBOUND_FORWARD_DNS: "{{ (DOCKER_IN_CONTAINER | bool) or ('microsoft' in lookup('file', '/proc/version') | lower) }}" + +UNBOUND_ONION_FORWARD_ADDR: >- + {{ (lookup('subnet_gateway') ~ '@' ~ lookup('config', 'svc-net-tor', 'services.tor.ports.local.dns')) + if ('svc-net-tor' in group_names and DEPLOYMENT_MODE == 'compose') else '' }} diff --git a/roles/svc-net-tor/files/test/onion_ports.py b/roles/svc-net-tor/files/test/onion_ports.py new file mode 100755 index 0000000000..b1985311ae --- /dev/null +++ b/roles/svc-net-tor/files/test/onion_ports.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Assert every TCP port the node onion forwards actually accepts a connection. + +The ports are the ones roles ask for by name under ports.onion, plus any +service opting in with exposed: true, which is what lookup('tor_ports') returns +and what torrc turns into HiddenServicePort lines. The unconditional SSH forward +is deliberately not probed: it targets a host service this role does not deploy, +so its absence says nothing about the forwards under test. + +The HTTP probe in test.sh only sees the reverse proxy's vhosts, so mail and +every other named port went unchecked, and from outside a forwarded port that +reaches nothing looks exactly like a port that never reached torrc at all. + +A port passes when the SOCKS proxy reports CONNECT success, which it does only +after Tor has established the stream to the target behind the onion. One pass +probes every port and only the failures are retried, so a green run costs one +connect per port and a red one cannot multiply the retry budget by the number of +healthy ports. + +Standard library only: this runs wherever the CLI test runs, not in a container +carrying a test toolchain. + +Env (rendered into test.env from templates/test.env.j2): + TOR_SOCKS SOCKS proxy, host:port + ONION_HOST the node onion host + ONION_PORTS space-separated forwarded ports; empty means there is + nothing to check + ONION_PORT_TIMEOUT seconds per connect attempt. 60 for the same reason the + domain probe uses it: Tor's SocksTimeout default is 120s + and abandoning earlier is destructive, because the next + attempt opens a fresh SOCKS connection and throws away + the descriptor fetches and rendezvous retries the last + one was still working through + ONION_PORT_RETRIES passes over the failing ports. Below the domain probe's + RETRIES because this phase multiplies by the port count, + and meta/tests.yml caps the whole CLI test + SLEEP_SECONDS wait between passes +""" + +from __future__ import annotations + +import os +import socket +import sys +import time + +_GREETING_NO_AUTH = b"\x05\x01\x00" +_GREETING_OK = b"\x05\x00" +_CONNECT_TO_DOMAIN = b"\x05\x01\x00\x03" +_REPLY_SUCCESS = 0x00 + + +class ProbeError(RuntimeError): + """A port did not answer, with the reason a reader needs to act on it.""" + + +def _required(name: str, hint: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise SystemExit(f"pass {name} as env ({hint})") + return value + + +def connect_through_socks( + proxy: str, host: str, port: int, timeout: float +) -> None: + """Open a SOCKS5 CONNECT to ``host:port`` and raise when it is refused.""" + proxy_host, _, proxy_port = proxy.rpartition(":") + try: + sock = socket.create_connection( + (proxy_host, int(proxy_port)), timeout=timeout + ) + except OSError as error: + raise ProbeError(f"socks proxy unreachable: {error}") from None + + with sock: + sock.settimeout(timeout) + try: + sock.sendall(_GREETING_NO_AUTH) + if sock.recv(2) != _GREETING_OK: + raise ProbeError("socks proxy refused the no-auth method") + target = host.encode() + sock.sendall( + _CONNECT_TO_DOMAIN + + bytes([len(target)]) + + target + + port.to_bytes(2, "big") + ) + reply = sock.recv(4) + except OSError as error: + raise ProbeError(f"socks handshake failed: {error}") from None + if len(reply) < 2: + raise ProbeError("socks proxy closed the connection") + if reply[1] != _REPLY_SUCCESS: + raise ProbeError(f"socks CONNECT failed with reply 0x{reply[1]:02x}") + + +def probe( + *, + proxy: str, + host: str, + ports: list[int], + timeout: float, + retries: int, + sleep_seconds: float, + sleeper=time.sleep, +) -> dict[int, str]: + """Return the ports still failing after the retries, with their reason.""" + pending = list(ports) + reasons: dict[int, str] = {} + for attempt in range(1, retries + 1): + still: list[int] = [] + for port in pending: + try: + connect_through_socks(proxy, host, port, timeout) + except ProbeError as error: + reasons[port] = str(error) + still.append(port) + else: + reasons.pop(port, None) + print(f"[OK] {host}:{port} connect") + pending = still + if not pending or attempt == retries: + break + print( + f"[INFO] pass {attempt + 1}/{retries} retries " + f"{len(pending)} port(s) in {sleep_seconds}s" + ) + sleeper(sleep_seconds) + return {port: reasons[port] for port in pending} + + +def main() -> int: + proxy = _required("TOR_SOCKS", "host:port of the node SOCKS proxy") + host = _required("ONION_HOST", "svc-net-tor services.tor.node") + retries = int(_required("ONION_PORT_RETRIES", "passes over the failing ports")) + sleep_seconds = float(_required("SLEEP_SECONDS", "wait between passes")) + timeout = float( + _required("ONION_PORT_TIMEOUT", "seconds per connect, flavor-dependent") + ) + ports = [int(entry) for entry in os.environ.get("ONION_PORTS", "").split()] + + if not ports: + print("[INFO] no onion-forwarded ports declared; nothing to probe") + return 0 + + print( + f"[INFO] probing {len(ports)} onion-forwarded port(s) on {host} " + f"via socks5://{proxy} ({timeout}s per connect)" + ) + failures = probe( + proxy=proxy, + host=host, + ports=ports, + timeout=timeout, + retries=retries, + sleep_seconds=sleep_seconds, + ) + for port, reason in failures.items(): + print(f"[FAIL] {host}:{port} {reason}") + print(f"[INFO] {len(ports)} port(s) probed, {len(failures)} failed") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/roles/svc-net-tor/files/test/test.sh b/roles/svc-net-tor/files/test/test.sh index 67bb92e08e..68677e034b 100755 --- a/roles/svc-net-tor/files/test/test.sh +++ b/roles/svc-net-tor/files/test/test.sh @@ -99,4 +99,7 @@ for domain in "${domains[@]}"; do done echo "[INFO] ${#domains[@]} probed, ${failed} failed" + +"$(dirname "${BASH_SOURCE[0]}")/onion_ports.py" || failed=$((failed + 1)) + [[ "${failed}" -eq 0 ]] diff --git a/roles/svc-net-tor/handlers/main.yml b/roles/svc-net-tor/handlers/main.yml new file mode 100644 index 0000000000..7f1b99b5a5 --- /dev/null +++ b/roles/svc-net-tor/handlers/main.yml @@ -0,0 +1,8 @@ +--- +- name: "๐Ÿ›ก๏ธ Apply the Tor egress guard" + listen: apply tor egress guard + ansible.builtin.systemd_service: + name: "{{ TOR_EGRESS_GUARD_UNIT | basename }}" + state: restarted + daemon_reload: true + become: true diff --git a/roles/svc-net-tor/lookup_plugins/tor_egress_clients.py b/roles/svc-net-tor/lookup_plugins/tor_egress_clients.py new file mode 100644 index 0000000000..51f5d48665 --- /dev/null +++ b/roles/svc-net-tor/lookup_plugins/tor_egress_clients.py @@ -0,0 +1,95 @@ +"""Lookup `tor_egress_clients`: who may use this node's Tor egress ports. + +Tor's DNSPort and TransPort carry no access policy of their own, unlike +SocksPort with its SocksPolicy, and a swarm-published port cannot be bound to a +single host address. The filter chain installed by ``tasks/router.yml`` is +therefore what bounds them, and this lookup is the list it admits. + +Deliberately not the three RFC1918 blocks: ``10.0.0.0/8`` would hand the +operator's whole LAN a resolver and a transparent proxy. Admitted is exactly the +address space this deployment declares for itself: + +* loopback, for the node's own dnsmasq, +* every base of ``NETWORK_DOCKER_ADDRESS_POOLS``, +* ``NETWORK_SWARM_GWBRIDGE_POOL``, the gateway bridge docker creates outside + those pools and without which overlay tasks lose their route out, +* ``NETWORK_ROLE_SUBNET_POOL`` and the blocks of the roles named in + ``NETWORK_ROLE_SUBNET_EXCEPTION_ROLES``, resolved through ``role_subnets`` so + the CIDR stays declared in the role that owns it. + +Usage: + + {{ lookup('tor_egress_clients') }} +""" + +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleError +from ansible.plugins.loader import lookup_loader +from ansible.plugins.lookup import LookupBase + +_REQUIRED = ( + "NETWORK_LOOPBACK_CIDR", + "NETWORK_DOCKER_ADDRESS_POOLS", + "NETWORK_SWARM_GWBRIDGE_POOL", + "NETWORK_ROLE_SUBNET_POOL", + "NETWORK_ROLE_SUBNET_EXCEPTION_ROLES", +) + + +def _pool_bases(pools: Any) -> list[str]: + if not isinstance(pools, (list, tuple)): + raise AnsibleError( + "tor_egress_clients: NETWORK_DOCKER_ADDRESS_POOLS must be a list" + ) + bases: list[str] = [] + for pool in pools: + base = pool.get("base") if isinstance(pool, dict) else None + if not base: + raise AnsibleError( + f"tor_egress_clients: address pool {pool!r} declares no base" + ) + bases.append(str(base).strip()) + return bases + + +class LookupModule(LookupBase): + def run( + self, + terms: list[Any] | None, + variables: dict[str, Any] | None = None, + **kwargs: Any, + ) -> list[Any]: + if terms: + raise AnsibleError("lookup('tor_egress_clients') expects no terms.") + variables = variables or getattr(self._templar, "available_variables", {}) or {} + missing = [name for name in _REQUIRED if variables.get(name) is None] + if missing: + raise AnsibleError( + "tor_egress_clients: missing group variable(s) " + f"{', '.join(missing)}; the guard must never be narrowed by a " + "value that silently defaulted" + ) + + excepted = lookup_loader.get( + "role_subnets", + loader=getattr(self, "_loader", None), + templar=getattr(self, "_templar", None), + ).run( + [variables["NETWORK_ROLE_SUBNET_EXCEPTION_ROLES"]], + variables=variables, + )[0] + + cidrs = [ + str(variables["NETWORK_LOOPBACK_CIDR"]).strip(), + *_pool_bases(variables["NETWORK_DOCKER_ADDRESS_POOLS"]), + str(variables["NETWORK_SWARM_GWBRIDGE_POOL"]).strip(), + str(variables["NETWORK_ROLE_SUBNET_POOL"]).strip(), + *excepted, + ] + seen: dict[str, None] = {} + for cidr in cidrs: + seen.setdefault(cidr, None) + return [list(seen)] diff --git a/roles/svc-net-tor/tasks/00_core.yml b/roles/svc-net-tor/tasks/00_core.yml index 7cf9dfc626..b7cbc59ccb 100644 --- a/roles/svc-net-tor/tasks/00_core.yml +++ b/roles/svc-net-tor/tasks/00_core.yml @@ -125,7 +125,9 @@ file: router.yml apply: become: true - when: TOR_EGRESS_ROUTER_ENABLED | bool + when: + - TOR_EGRESS_ROUTER_ENABLED | bool + - DEPLOYMENT_MODE == 'compose' or (IS_STACK_HOST | bool) - name: "๐Ÿง… Verify the node onion is reachable" ansible.builtin.include_tasks: onion_reach.yml diff --git a/roles/svc-net-tor/tasks/router.yml b/roles/svc-net-tor/tasks/router.yml index 3ab0421d00..51a2fcbe14 100644 --- a/roles/svc-net-tor/tasks/router.yml +++ b/roles/svc-net-tor/tasks/router.yml @@ -97,3 +97,92 @@ jump: REDIRECT to_ports: "{{ TOR_TRANS_PORT | string }}" comment: "Tor egress router: .onion virtual range -> TransPort" + +- name: "๐Ÿ”€ Egress router | Send container .onion traffic to the bridge TransPort" + when: TOR_CONTAINER_DNS_HOST | length > 0 + ansible.builtin.iptables: + table: nat + chain: PREROUTING + protocol: tcp + destination: "{{ TOR_EGRESS_VIRTUAL_NET_IPV4 }}" + jump: DNAT + to_destination: "{{ TOR_CONTAINER_DNS_HOST }}:{{ TOR_TRANS_PORT }}" + comment: "Tor egress router: container .onion virtual range -> TransPort" + +- name: "๐Ÿ”€ Egress router | Prove the container redirect is installed" + when: TOR_CONTAINER_DNS_HOST | length > 0 + ansible.builtin.command: + argv: + - iptables + - -t + - nat + - -C + - PREROUTING + - -p + - tcp + - -d + - "{{ TOR_EGRESS_VIRTUAL_NET_IPV4 }}" + - -j + - DNAT + - --to-destination + - "{{ TOR_CONTAINER_DNS_HOST }}:{{ TOR_TRANS_PORT }}" + changed_when: false + +- name: "๐Ÿ›ก๏ธ Egress guard | Stage the guard applier" + when: DEPLOYMENT_MODE == 'swarm' + ansible.builtin.template: + src: tor-egress-guard.sh.j2 + dest: "{{ TOR_EGRESS_GUARD_SCRIPT }}" + owner: root + group: root + mode: "0750" + notify: apply tor egress guard + +- name: "๐Ÿ›ก๏ธ Egress guard | Install the unit that reapplies it after docker" + when: DEPLOYMENT_MODE == 'swarm' + ansible.builtin.template: + src: tor-egress-guard.service.j2 + dest: "{{ TOR_EGRESS_GUARD_UNIT }}" + owner: root + group: root + mode: "0644" + notify: apply tor egress guard + +- name: "๐Ÿ›ก๏ธ Egress guard | Enable the guard across reboots" + when: DEPLOYMENT_MODE == 'swarm' + ansible.builtin.systemd_service: + name: "{{ TOR_EGRESS_GUARD_UNIT | basename }}" + enabled: true + state: started + daemon_reload: true + +- name: "๐Ÿ›ก๏ธ Egress guard | Flush handlers so the guard is live before checking" + when: DEPLOYMENT_MODE == 'swarm' + ansible.builtin.meta: flush_handlers + +- name: "๐Ÿ›ก๏ธ Egress guard | Prove the guard rejects everything it must" + when: DEPLOYMENT_MODE == 'swarm' + ansible.builtin.command: + argv: + - iptables + - -C + - "{{ TOR_EGRESS_GUARD_CHAIN }}" + - -j + - DROP + changed_when: false + +- name: "๐Ÿ›ก๏ธ Egress guard | Prove every guarded port reaches the guard" + when: DEPLOYMENT_MODE == 'swarm' + ansible.builtin.command: + argv: + - iptables + - -C + - "{{ item.0 }}" + - -p + - "{{ item.1.protocol }}" + - --dport + - "{{ item.1.port | string }}" + - -j + - "{{ TOR_EGRESS_GUARD_CHAIN }}" + changed_when: false + loop: "{{ TOR_EGRESS_GUARD_HOOK_CHAINS | product(TOR_EGRESS_GUARDED_PORTS) | list }}" diff --git a/roles/svc-net-tor/templates/compose.yml.j2 b/roles/svc-net-tor/templates/compose.yml.j2 index 59ddd84fdb..81c0ffdaec 100644 --- a/roles/svc-net-tor/templates/compose.yml.j2 +++ b/roles/svc-net-tor/templates/compose.yml.j2 @@ -14,7 +14,11 @@ {% if DEPLOYMENT_MODE == 'swarm' %} {{ lookup('container_ports', {'service': 'tor', 'protocol': 'socks', 'mode': 'host', - 'container': 'same'}) | indent(4) }} + 'container': 'same'}, + {'service': 'tor', 'protocol': 'dns', 'mode': 'host', + 'container': 'same', 'transport': ['udp', 'tcp']}, + {'service': 'tor', 'protocol': 'trans', 'mode': 'host', + 'container': 'same', 'transport': 'tcp'}) | indent(4) }} {{ lookup('container_networks') | indent(4) }} {% else %} network_mode: "host" diff --git a/roles/svc-net-tor/templates/test.env.j2 b/roles/svc-net-tor/templates/test.env.j2 index 10028ca3a0..0b58f45bb0 100644 --- a/roles/svc-net-tor/templates/test.env.j2 +++ b/roles/svc-net-tor/templates/test.env.j2 @@ -1,4 +1,8 @@ NGINX_SERVERS_DIR={{ lookup('nginx', 'directories.configuration.servers') }} +ONION_HOST={{ lookup('config', 'svc-net-tor', 'services.tor.node') }} +ONION_PORTS={{ lookup('tor_ports') | map(attribute='onion_port') | unique | sort | join(' ') }} +ONION_PORT_RETRIES=2 +ONION_PORT_TIMEOUT=60 RETRIES=5 SLEEP_SECONDS=10 TOR_DNSMASQ_CONF={{ TOR_DNSMASQ_CONF }} diff --git a/roles/svc-net-tor/templates/tor-egress-guard.service.j2 b/roles/svc-net-tor/templates/tor-egress-guard.service.j2 new file mode 100644 index 0000000000..fae8f0cdc6 --- /dev/null +++ b/roles/svc-net-tor/templates/tor-egress-guard.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Infinito.Nexus Tor egress guard +Documentation=file://{{ TOR_EGRESS_GUARD_SCRIPT }} +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart={{ TOR_EGRESS_GUARD_SCRIPT }} + +[Install] +WantedBy=multi-user.target diff --git a/roles/svc-net-tor/templates/tor-egress-guard.sh.j2 b/roles/svc-net-tor/templates/tor-egress-guard.sh.j2 new file mode 100644 index 0000000000..c8892062a4 --- /dev/null +++ b/roles/svc-net-tor/templates/tor-egress-guard.sh.j2 @@ -0,0 +1,21 @@ +#!/bin/bash +# Managed by Infinito.Nexus (roles/svc-net-tor) โ€” do not edit by hand. +set -euo pipefail + +CHAIN={{ TOR_EGRESS_GUARD_CHAIN | quote }} + +iptables -N "${CHAIN}" 2>/dev/null || true +iptables -F "${CHAIN}" + +{% for cidr in TOR_EGRESS_CLIENT_CIDRS %} +iptables -A "${CHAIN}" -s {{ cidr | quote }} -j ACCEPT +{% endfor %} +iptables -A "${CHAIN}" -j DROP + +{% for chain in TOR_EGRESS_GUARD_HOOK_CHAINS %} +iptables -N {{ chain | quote }} 2>/dev/null || true +{% for entry in TOR_EGRESS_GUARDED_PORTS %} +iptables -C {{ chain | quote }} -p {{ entry.protocol | quote }} --dport {{ entry.port | quote }} -j "${CHAIN}" 2>/dev/null \ + || iptables -I {{ chain | quote }} -p {{ entry.protocol | quote }} --dport {{ entry.port | quote }} -j "${CHAIN}" +{% endfor %} +{% endfor %} diff --git a/roles/svc-net-tor/templates/torrc.j2 b/roles/svc-net-tor/templates/torrc.j2 index b2167aeea9..bcaf9564ae 100644 --- a/roles/svc-net-tor/templates/torrc.j2 +++ b/roles/svc-net-tor/templates/torrc.j2 @@ -4,10 +4,9 @@ Log notice stdout DataDirectory {{ lookup('volume', 'svc-net-tor', 'tor_data').target }} SocksPort {{ lookup('tor_socks', '0.0.0.0') }} -SocksPolicy accept 127.0.0.0/8 -SocksPolicy accept 10.0.0.0/8 -SocksPolicy accept 172.16.0.0/12 -SocksPolicy accept 192.168.0.0/16 +{% for _cidr in NETWORK_INTERNAL_CIDRS %} +SocksPolicy accept {{ _cidr }} +{% endfor %} SocksPolicy reject * ControlPort 127.0.0.1:9051 CookieAuthentication 1 @@ -24,12 +23,20 @@ UseEntryGuards 0 AutomapHostsOnResolve 1 AutomapHostsSuffixes .onion VirtualAddrNetworkIPv4 {{ TOR_EGRESS_VIRTUAL_NET_IPV4 }} +{% if DEPLOYMENT_MODE == 'swarm' %} +TransPort 0.0.0.0:{{ TOR_TRANS_PORT }} +DNSPort 0.0.0.0:{{ TOR_DNS_PORT }} +{% else %} TransPort 127.0.0.1:{{ TOR_TRANS_PORT }} DNSPort 127.0.0.1:{{ TOR_DNS_PORT }} +{% if TOR_CONTAINER_DNS_HOST | length > 0 %} +TransPort {{ TOR_CONTAINER_DNS_HOST }}:{{ TOR_TRANS_PORT }} +DNSPort {{ TOR_CONTAINER_DNS_HOST }}:{{ TOR_DNS_PORT }} +{% endif %} +{% endif %} {% endif %} HiddenServiceDir /var/lib/tor/infinito/node/ HiddenServiceVersion 3 -HiddenServicePort 80 {{ TOR_HS_BACKEND_HOST }}:80 {% for _p in TOR_ONION_EXTRA_PORTS %} HiddenServicePort {{ _p.onion_port }} {{ TOR_HS_BACKEND_HOST }}:{{ _p.target.rsplit(':', 1) | last }} {% endfor %} diff --git a/roles/svc-net-tor/vars/main.yml b/roles/svc-net-tor/vars/main.yml index 6eb3bb1f87..9f0c392b3f 100644 --- a/roles/svc-net-tor/vars/main.yml +++ b/roles/svc-net-tor/vars/main.yml @@ -12,6 +12,21 @@ TOR_FLAVOR: "{{ lookup('config', application_id, 'services.tor.flavor') }}" TOR_TRANS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.trans') }}" TOR_DNS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.dns') }}" TOR_DNSMASQ_CONF: "/etc/dnsmasq.d/tor-onion.conf" + +TOR_EGRESS_GUARD_CHAIN: INFINITO_TOR_EGRESS +TOR_EGRESS_GUARD_SCRIPT: "/usr/local/bin/tor-egress-guard.sh" +TOR_EGRESS_GUARD_UNIT: "/etc/systemd/system/tor-egress-guard.service" +TOR_EGRESS_GUARD_HOOK_CHAINS: + - INPUT + - DOCKER-USER +TOR_EGRESS_CLIENT_CIDRS: "{{ lookup('tor_egress_clients') }}" +TOR_EGRESS_GUARDED_PORTS: + - port: "{{ TOR_TRANS_PORT }}" + protocol: tcp + - port: "{{ TOR_DNS_PORT }}" + protocol: tcp + - port: "{{ TOR_DNS_PORT }}" + protocol: udp TOR_DNSMASQ_DROPIN: "/etc/systemd/system/dnsmasq.service.d/infinito-restart.conf" # nocheck: host-local-fact meta/services.yml pins this role to the manager, so the fact describes the node the hidden service forwards on TOR_CONTAINER_DNS_HOST: "{{ ansible_facts['docker0']['ipv4']['address'] | default('', true) }}" diff --git a/tests/unit/python/roles/svc-net-tor/test_egress_guard_script.py b/tests/unit/python/roles/svc-net-tor/test_egress_guard_script.py new file mode 100644 index 0000000000..2b0fcd7b35 --- /dev/null +++ b/tests/unit/python/roles/svc-net-tor/test_egress_guard_script.py @@ -0,0 +1,110 @@ +"""The rendered Tor egress guard rebuilds its chain instead of adding to it. + +The first version of this guard appended: it created the chain when absent and +appended one ACCEPT per admitted range. Narrowing the admitted set then did +nothing, because the retired range had no removal step and the new one landed +below the DROP, where no packet reaches it. The chain could be loosened and +never tightened, which is the wrong direction for a rule that is the only +boundary two policy-less ports have. + +What keeps that from coming back is an order: flush, then admit, then drop, then +hook. These tests pin it against the rendered script, because the ordering is +invisible in the template's individual lines. +""" + +from __future__ import annotations + +import shlex +import unittest +from pathlib import Path + +import jinja2 + +from . import PROJECT_ROOT + +TEMPLATE = ( + PROJECT_ROOT / "roles" / "svc-net-tor" / "templates" / "tor-egress-guard.sh.j2" +) + +CHAIN = "INFINITO_TOR_EGRESS" +CLIENTS = ["127.0.0.0/8", "10.208.0.0/12"] +HOOKS = ["INPUT", "DOCKER-USER"] +PORTS = [ + {"port": "9040", "protocol": "tcp"}, + {"port": "9053", "protocol": "udp"}, +] + + +def _render() -> list[str]: + environment = jinja2.Environment( + keep_trailing_newline=True, + autoescape=False, # noqa: S701 shell, not markup + ) + environment.filters["quote"] = shlex.quote + rendered = environment.from_string( + Path(TEMPLATE).read_text(encoding="utf-8") + ).render( + TOR_EGRESS_GUARD_CHAIN=CHAIN, + TOR_EGRESS_CLIENT_CIDRS=CLIENTS, + TOR_EGRESS_GUARD_HOOK_CHAINS=HOOKS, + TOR_EGRESS_GUARDED_PORTS=PORTS, + ) + return [line.strip() for line in rendered.splitlines() if line.strip()] + + +def _index_of(lines: list[str], needle: str) -> int: + for position, line in enumerate(lines): + if needle in line: + return position + raise AssertionError(f"the rendered guard never {needle!r}:\n" + "\n".join(lines)) + + +class TestEgressGuardScript(unittest.TestCase): + def test_the_chain_is_flushed_before_it_is_filled(self) -> None: + lines = _render() + flush = _index_of(lines, '-F "${CHAIN}"') + first_admit = _index_of(lines, f"-s {CLIENTS[0]}") + self.assertLess( + flush, + first_admit, + "the chain must be flushed before the admitted ranges are added, or a " + "narrowed client list leaves the old range in place", + ) + + def test_every_admitted_range_precedes_the_drop(self) -> None: + lines = _render() + drop = _index_of(lines, "-j DROP") + for cidr in CLIENTS: + with self.subTest(cidr=cidr): + self.assertLess( + _index_of(lines, f"-s {cidr}"), + drop, + f"{cidr} is admitted below the DROP, where it never matches", + ) + + def test_the_hooks_come_after_the_chain_is_complete(self) -> None: + """A hook installed first would route packets into a half-built chain.""" + lines = _render() + drop = _index_of(lines, "-j DROP") + for chain in HOOKS: + for entry in PORTS: + with self.subTest(chain=chain, port=entry["port"]): + hook = _index_of(lines, f"-I {chain} -p {entry['protocol']}") + self.assertGreater(hook, drop) + + def test_every_admitted_range_and_port_is_rendered(self) -> None: + lines = _render() + body = "\n".join(lines) + for cidr in CLIENTS: + self.assertIn(cidr, body) + for chain in HOOKS: + for entry in PORTS: + self.assertIn( + f"--dport {entry['port']}", + body, + f"{chain} never guards {entry['protocol']}/{entry['port']}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/roles/svc-net-tor/test_onion_ports_probe.py b/tests/unit/python/roles/svc-net-tor/test_onion_ports_probe.py new file mode 100644 index 0000000000..e9ab2bb600 --- /dev/null +++ b/tests/unit/python/roles/svc-net-tor/test_onion_ports_probe.py @@ -0,0 +1,162 @@ +"""The onion port probe speaks SOCKS5 and reports only real successes. + +The handshake used to live in a heredoc inside a shell script, where no linter +and no test could reach it: forty lines assembling protocol bytes, checked by +nothing. These tests run it against a fake proxy that answers by script, so the +bytes on the wire and the verdict drawn from the reply are both pinned. +""" + +from __future__ import annotations + +import importlib.util +import socket +import threading +import unittest + +from . import PROJECT_ROOT + + +def _load_probe(): + path = PROJECT_ROOT / "roles" / "svc-net-tor" / "files" / "test" / "onion_ports.py" + spec = importlib.util.spec_from_file_location("onion_ports_probe", str(path)) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +PROBE = _load_probe() + + +class _FakeSocks(threading.Thread): + """A one-shot SOCKS5 proxy that replies with a scripted CONNECT code.""" + + def __init__(self, reply_code: int | None, greeting: bytes = b"\x05\x00") -> None: + super().__init__(daemon=True) + self._reply_code = reply_code + self._greeting = greeting + self._server = socket.socket() + self._server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._server.bind(("127.0.0.1", 0)) + self._server.listen(1) + self.request = b"" + + @property + def endpoint(self) -> str: + host, port = self._server.getsockname() + return f"{host}:{port}" + + def run(self) -> None: + conn, _ = self._server.accept() + with conn: + conn.recv(3) + conn.sendall(self._greeting) + if self._greeting != b"\x05\x00": + return + self.request = conn.recv(512) + if self._reply_code is None: + return + conn.sendall(bytes([0x05, self._reply_code, 0x00, 0x01])) + self._server.close() + + +class TestConnectThroughSocks(unittest.TestCase): + def test_a_success_reply_passes_and_carries_the_target(self) -> None: + proxy = _FakeSocks(reply_code=0x00) + proxy.start() + PROBE.connect_through_socks(proxy.endpoint, "example.onion", 25, 5) + proxy.join(timeout=5) + self.assertTrue( + proxy.request.startswith(b"\x05\x01\x00\x03"), + f"the CONNECT request is malformed: {proxy.request!r}", + ) + self.assertIn(b"example.onion", proxy.request) + self.assertTrue( + proxy.request.endswith((25).to_bytes(2, "big")), + "the port must travel big-endian in the last two bytes", + ) + + def test_a_refusal_reply_fails(self) -> None: + proxy = _FakeSocks(reply_code=0x05) + proxy.start() + with self.assertRaises(PROBE.ProbeError) as raised: + PROBE.connect_through_socks(proxy.endpoint, "example.onion", 25, 5) + self.assertIn("0x05", str(raised.exception)) + + def test_a_refused_auth_method_fails(self) -> None: + proxy = _FakeSocks(reply_code=None, greeting=b"\x05\xff") + proxy.start() + with self.assertRaises(PROBE.ProbeError): + PROBE.connect_through_socks(proxy.endpoint, "example.onion", 25, 5) + + def test_a_closed_connection_fails(self) -> None: + proxy = _FakeSocks(reply_code=None) + proxy.start() + with self.assertRaises(PROBE.ProbeError): + PROBE.connect_through_socks(proxy.endpoint, "example.onion", 25, 5) + + def test_an_unreachable_proxy_fails(self) -> None: + with self.assertRaises(PROBE.ProbeError) as raised: + PROBE.connect_through_socks("127.0.0.1:1", "example.onion", 25, 1) + self.assertIn("unreachable", str(raised.exception)) + + +class TestProbe(unittest.TestCase): + def test_only_the_failures_are_retried(self) -> None: + attempts: list[int] = [] + + def _connect(_proxy, _host, port, _timeout): + attempts.append(port) + if port == 25: + raise PROBE.ProbeError("still down") + + original = PROBE.connect_through_socks + PROBE.connect_through_socks = _connect + try: + failures = PROBE.probe( + proxy="p:9050", + host="example.onion", + ports=[25, 143], + timeout=1, + retries=3, + sleep_seconds=0, + sleeper=lambda _seconds: None, + ) + finally: + PROBE.connect_through_socks = original + + self.assertEqual(failures, {25: "still down"}) + self.assertEqual( + attempts.count(143), + 1, + "a healthy port must not be probed again on a retry pass", + ) + self.assertEqual(attempts.count(25), 3) + + def test_a_port_that_recovers_is_not_reported(self) -> None: + seen: list[int] = [] + + def _connect(_proxy, _host, port, _timeout): + seen.append(port) + if len(seen) == 1: + raise PROBE.ProbeError("first pass fails") + + original = PROBE.connect_through_socks + PROBE.connect_through_socks = _connect + try: + failures = PROBE.probe( + proxy="p:9050", + host="example.onion", + ports=[25], + timeout=1, + retries=2, + sleep_seconds=0, + sleeper=lambda _seconds: None, + ) + finally: + PROBE.connect_through_socks = original + + self.assertEqual(failures, {}) + + +if __name__ == "__main__": + unittest.main() From 99f7e0384522c8cf524d7fcee8ae3d6279302919 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 18:09:43 +0200 Subject: [PATCH 043/197] feat(sys-service): carry unit ordering, an Install section and a user scope The generic unit template could express neither an ordering dependency nor an Install section, so a role needing either shipped its own templates/systemctl.service.j2 and re-answered the daemon reload, the enable and the flush behaviour by hand. Add After, Before, Wants, RemainAfterExit and a WantedBy driven Install section, all inert at their defaults, and route the unit path, its owner, become and the systemctl probes through a system_service_scope so a per-login unit lands under the user's own systemd directory. 00_core is gated on the system scope rather than only its prune and lockdown steps: it sets run_once_sys_service, so a user-scope caller running first would otherwise skip the lockdown for every system-scope role behind it in the same play. Rendering the previous template against the same inputs drops no directive and adds none; the only difference is the blank line an empty directive used to leave behind. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/05_paths.yml | 1 + .../handlers/refresh_systemctl_service.yml | 24 +++++-------------- roles/sys-service/tasks/06_service.yml | 7 +++--- roles/sys-service/tasks/07_timer.yml | 10 ++++++++ roles/sys-service/tasks/main.yml | 4 +++- .../templates/systemctl.service.j2 | 24 ++++++++++++++++--- roles/sys-service/vars/main.yml | 11 +++++++++ 7 files changed, 56 insertions(+), 25 deletions(-) diff --git a/group_vars/all/05_paths.yml b/group_vars/all/05_paths.yml index 270384ce19..d0a2ab7734 100644 --- a/group_vars/all/05_paths.yml +++ b/group_vars/all/05_paths.yml @@ -5,6 +5,7 @@ DIR_SCRIPTS_ROOT: "/opt/scripts/" DIR_SCRIPTS_SYSTEMCTL: "{{ [ DIR_SCRIPTS_ROOT, 'systemctl' ] | path_join }}" DIR_COMPOSITIONS: "/opt/compose/" DIR_SYSTEM_SERVICE: "/etc/systemd/system" +DIR_SYSTEM_SERVICE_USER: "$HOME/.config/systemd/user" DIR_COMPOSE_PULL_LOCK: "/run/ansible/compose-pull/" DIR_BIN: "/usr/bin" diff --git a/roles/sys-service/handlers/refresh_systemctl_service.yml b/roles/sys-service/handlers/refresh_systemctl_service.yml index 879a7dd6e9..ac19505e55 100644 --- a/roles/sys-service/handlers/refresh_systemctl_service.yml +++ b/roles/sys-service/handlers/refresh_systemctl_service.yml @@ -2,9 +2,10 @@ - name: "๐Ÿ”ง Enable systemctl service" systemd: name: "{{ system_service_name }}" + scope: "{{ system_service_scope }}" enabled: true daemon_reload: true - become: true + become: "{{ system_service_unit_become }}" # rationale: async; opt-out per caller, a role that reads the unit state sets system_service_force_linear_sync async: "{{ system_service_async }}" poll: "{{ system_service_poll }}" @@ -12,8 +13,9 @@ - name: "๐Ÿ”ง Set systemctl service state" systemd: name: "{{ system_service_name }}" + scope: "{{ system_service_scope }}" state: "{{ system_service_state }}" - become: true + become: "{{ system_service_unit_become }}" # rationale: async; the Result gate below only runs meaningfully for callers that set system_service_force_linear_sync async: "{{ system_service_async }}" poll: "{{ system_service_poll }}" @@ -23,28 +25,14 @@ - name: ๐Ÿ” sys-service | Read unit Result (oneshot-safe) ansible.builtin.command: - argv: - - systemctl - - --no-pager - - show - - -p - - Result - - --value - - "{{ system_service_name }}" + argv: "{{ system_service_systemctl_show + ['Result', system_service_name] }}" register: system_service_result changed_when: false failed_when: false - name: ๐Ÿ” sys-service | Read unit ExecMainStatus (oneshot-safe) ansible.builtin.command: - argv: - - systemctl - - --no-pager - - show - - -p - - ExecMainStatus - - --value - - "{{ system_service_name }}" + argv: "{{ system_service_systemctl_show + ['ExecMainStatus', system_service_name] }}" register: system_service_exec_status changed_when: false failed_when: false diff --git a/roles/sys-service/tasks/06_service.yml b/roles/sys-service/tasks/06_service.yml index 639c21ba32..a89a63ad6d 100644 --- a/roles/sys-service/tasks/06_service.yml +++ b/roles/sys-service/tasks/06_service.yml @@ -28,10 +28,11 @@ - name: "๐Ÿ”ง setup systemctl '{{ system_service_id }}'" template: # nocheck: stack-host-template (per-host systemd service unit) src: "{{ system_service_template_src }}" - dest: "{{ [ DIR_SYSTEM_SERVICE, system_service_name ] | path_join }}" - owner: root - group: root + dest: "{{ [ system_service_unit_dir, system_service_name ] | path_join }}" + owner: "{{ system_service_unit_owner }}" + group: "{{ system_service_unit_owner }}" mode: "0644" + become: "{{ system_service_unit_become }}" notify: "{{ 'reload-system-daemon' if system_service_is_template | bool else 'refresh-systemctl-service' }}" - name: ๐Ÿ”„ refresh systemctl service when SYS_SERVICE_ALL_ENABLE diff --git a/roles/sys-service/tasks/07_timer.yml b/roles/sys-service/tasks/07_timer.yml index 1701ebaaf3..4814d6ad92 100644 --- a/roles/sys-service/tasks/07_timer.yml +++ b/roles/sys-service/tasks/07_timer.yml @@ -1,4 +1,14 @@ --- +- name: ๐Ÿ›‘ Fail if a timer is asked for outside the system scope + assert: + that: + - system_service_scope == 'system' + fail_msg: >- + '{{ system_service_id }}' asks for a timer in the '{{ system_service_scope }}' + scope, but sys-timer writes into {{ DIR_SYSTEM_SERVICE }} and drives the unit + in the system scope, so the timer would look for a service that only exists + for the user. + - name: ๐Ÿ›‘ Fail if system_service_id contains "@" assert: that: diff --git a/roles/sys-service/tasks/main.yml b/roles/sys-service/tasks/main.yml index ff13725fc0..c759025647 100644 --- a/roles/sys-service/tasks/main.yml +++ b/roles/sys-service/tasks/main.yml @@ -6,7 +6,9 @@ when: run_once_sys_service is defined - include_tasks: 00_core.yml - when: run_once_sys_service is not defined + when: + - run_once_sys_service is not defined + - system_service_scope == 'system' - name: "โ–ถ๏ธ Execute service routines for '{{ system_service_id }}'" block: diff --git a/roles/sys-service/templates/systemctl.service.j2 b/roles/sys-service/templates/systemctl.service.j2 index 4ec2bcb6b8..5f9a7dd7d1 100644 --- a/roles/sys-service/templates/systemctl.service.j2 +++ b/roles/sys-service/templates/systemctl.service.j2 @@ -1,6 +1,16 @@ [Unit] Description={{ SOFTWARE_NAME }} - Service for role '{{ system_service_id }}' -{{ system_service_tpl_on_failure | systemd_directive('OnFailure') }} +{% for key, val in [ + ('OnFailure', system_service_tpl_on_failure), + ('After', system_service_tpl_after), + ('Before', system_service_tpl_before), + ('Wants', system_service_tpl_wants), +] %} +{% set directive = val | systemd_directive(key) %} +{% if directive %} +{{ directive }} +{% endif %} +{% endfor %} StartLimitIntervalSec=60 StartLimitBurst=3 OnFailureJobMode=replace-irreversibly @@ -10,13 +20,21 @@ Type={{ system_service_tpl_type }} {% for _environment in system_service_tpl_environment %} Environment={{ _environment }} {% endfor %} - {% for key, val in [ + ('RemainAfterExit', system_service_tpl_remain_after_exit), ('TimeoutStartSec', system_service_tpl_timeout_start_sec), ('ExecStartPre', system_service_tpl_exec_start_pre), ('ExecStart', system_service_tpl_exec_start), ('ExecStartPost', system_service_tpl_exec_start_post), ('RuntimeMaxSec', system_service_tpl_runtime), ] %} -{{ val | systemd_directive(key) }} +{% set directive = val | systemd_directive(key) %} +{% if directive %} +{{ directive }} +{% endif %} {% endfor %} +{% if system_service_tpl_wanted_by | length > 0 %} + +[Install] +WantedBy={{ system_service_tpl_wanted_by }} +{% endif %} diff --git a/roles/sys-service/vars/main.yml b/roles/sys-service/vars/main.yml index 2a91e808bf..24b8e3fae9 100644 --- a/roles/sys-service/vars/main.yml +++ b/roles/sys-service/vars/main.yml @@ -4,6 +4,12 @@ UNIT_SUFFIX_REMOVER_PACKAGE: "unsure" system_service_name: "{{ lookup('unit_name', system_service_id) }}" system_service_is_template: "{{ system_service_id.endswith('@') }}" +system_service_scope: "system" # 'system' for root units, 'user' for per-login units +system_service_unit_dir: "{{ DIR_SYSTEM_SERVICE if system_service_scope == 'system' else DIR_SYSTEM_SERVICE_USER }}" +system_service_unit_owner: "{{ 'root' if system_service_scope == 'system' else omit }}" +system_service_unit_become: "{{ system_service_scope == 'system' }}" +system_service_systemctl_show: "{{ ['systemctl'] + (['--user'] if system_service_scope == 'user' else []) + ['--no-pager', 'show', '--value', '-p'] }}" + system_service_role_name: "{{ system_service_id | regex_replace('@','') }}" system_service_role_dir: "{{ lookup('path_absolute', 'roles', system_service_role_name) }}" system_service_script_dir: "{{ [ DIR_SCRIPTS_SYSTEMCTL, system_service_id ] | path_join }}" @@ -25,7 +31,12 @@ system_service_script_inter: "{{ '/bin/bash' if system_service_script_type == 's system_service_script_exec: "{{ system_service_script_inter }} {{ system_service_id | get_service_script_path( system_service_script_type ) }}" system_service_tpl_on_failure: "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}" +system_service_tpl_after: "" +system_service_tpl_before: "" +system_service_tpl_wants: "" system_service_tpl_type: "oneshot" +system_service_tpl_remain_after_exit: "" +system_service_tpl_wanted_by: "" # Empty installs no [Install] section, which leaves the unit static system_service_tpl_environment: [] # Environment variables system_service_tpl_runtime: "{{ '' if system_service_tpl_type == 'oneshot' else SYS_SERVICE_DEFAULT_RUNTIME }}" system_service_tpl_exec_start_pre: "" From bf32f57b85dc667014484256b7655d0c5156c312 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 20:02:36 +0200 Subject: [PATCH 044/197] feat(svc-net-firewall): own the packet filter as one nftables table per role svc-net-tor rendered its own guard script, its own unit and its own handler, and hooked the rules into DOCKER-USER with an insert, because a rule appended to a chain docker owns lands behind the RETURN docker puts there. The script flushed its chain and refilled it rule by rule, so the boundary was gone for the duration of the refill. A role that needs packet filter rules now writes templates/nftables.conf.j2 declaring its own table and includes svc-net-firewall, which renders it to /etc/nftables.d/.conf, rejects it with nft -c, loads it, reads the table back and refuses one that carries no chain. Deployed on its own the role installs nftables and the ruleset that loads every fragment at boot. That ruleset is a distribution decision, so its path follows os_family: RedHat reads /etc/sysconfig/nftables.conf, the others /etc/nftables.conf. The unit is only enabled, never started, because on Debian its ExecStop flushes the whole ruleset, docker's tables included. One table per role removes the shared path: a fragment replaces its own table and touches no other, and docker never touches either. nft -f on an existing table merges, so each fragment creates, deletes and redeclares its table in one transaction. The tests load every fragment twice into a throwaway network namespace and require the listing to match, because a merging fragment loads twice without error and shows up only as a duplicated rule. The tor guard now applies in both deploy modes. It was pinned to swarm while torrc binds TransPort and DNSPort on the bridge address in compose too, so on a single-host install those ports answered every on-link neighbour. The guard drops rather than admits, so nothing short-circuits another table, and ip saddr carries an implicit ipv4 dependency, which leaves the ipv6 exposure exactly where the iptables rules left it. svc-net-tor reads its own chain back and refuses a guard that drops nothing. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 1 + docs/requirements/035-mcp-proxy-expansion.md | 2 +- roles/svc-net-firewall/README.md | 90 ++++++++ roles/svc-net-firewall/meta/info.yml | 5 + roles/svc-net-firewall/meta/main.yml | 27 +++ roles/svc-net-firewall/meta/packages.yml | 8 + roles/svc-net-firewall/meta/services.yml | 16 ++ roles/svc-net-firewall/tasks/01_host.yml | 32 +++ roles/svc-net-firewall/tasks/02_fragment.yml | 43 ++++ roles/svc-net-firewall/tasks/main.yml | 8 + roles/svc-net-firewall/templates/main.nft.j2 | 3 + roles/svc-net-firewall/vars/main.yml | 15 ++ roles/svc-net-tor/files/test/onion_ports.py | 8 +- roles/svc-net-tor/handlers/main.yml | 8 - .../lookup_plugins/tor_egress_clients.py | 90 ++++---- roles/svc-net-tor/tasks/router.yml | 109 ++------- roles/svc-net-tor/templates/nftables.conf.j2 | 38 ++++ .../templates/tor-egress-guard.service.j2 | 13 -- .../templates/tor-egress-guard.sh.j2 | 21 -- roles/svc-net-tor/vars/main.yml | 8 +- .../jinja/test_variable_definitions.py | 2 + .../jinja/test_shell_templates_are_valid.py | 10 +- .../python/roles/svc-net-firewall/__init__.py | 3 + .../roles/svc-net-firewall/test_fragments.py | 208 ++++++++++++++++++ .../svc-net-tor/test_egress_guard_script.py | 110 --------- 25 files changed, 567 insertions(+), 311 deletions(-) create mode 100644 roles/svc-net-firewall/README.md create mode 100644 roles/svc-net-firewall/meta/info.yml create mode 100644 roles/svc-net-firewall/meta/main.yml create mode 100644 roles/svc-net-firewall/meta/packages.yml create mode 100644 roles/svc-net-firewall/meta/services.yml create mode 100644 roles/svc-net-firewall/tasks/01_host.yml create mode 100644 roles/svc-net-firewall/tasks/02_fragment.yml create mode 100644 roles/svc-net-firewall/tasks/main.yml create mode 100644 roles/svc-net-firewall/templates/main.nft.j2 create mode 100644 roles/svc-net-firewall/vars/main.yml delete mode 100644 roles/svc-net-tor/handlers/main.yml create mode 100644 roles/svc-net-tor/templates/nftables.conf.j2 delete mode 100644 roles/svc-net-tor/templates/tor-egress-guard.service.j2 delete mode 100644 roles/svc-net-tor/templates/tor-egress-guard.sh.j2 create mode 100644 tests/unit/python/roles/svc-net-firewall/__init__.py create mode 100644 tests/unit/python/roles/svc-net-firewall/test_fragments.py delete mode 100644 tests/unit/python/roles/svc-net-tor/test_egress_guard_script.py diff --git a/README.md b/README.md index 9963b1ec95..873f7bf390 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ Every invokable role in the tested lifecycle envelope, with its upstream homepag | [Fediwall](roles/web-app-fediwall/) | beta | Self-hosted Fediverse wall for showing recent public posts by hashtag or account in a live masonry grid | [๐ŸŒ](https://github.com/defnull/fediwall) [๐Ÿ› ๏ธ](roles/web-app-fediwall/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Friendica](roles/web-app-friendica/), [Mastodon](roles/web-app-mastodon/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | | [Fider](roles/web-app-fider/) | beta | Open-source community feedback platform for collecting, voting on, and prioritizing feature requests and ideas | [๐ŸŒ](https://www.fider.io/) [๐Ÿ› ๏ธ](roles/web-app-fider/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Hermes Agent](roles/web-app-hermes/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Open WebUI](roles/web-app-openwebui/), [OpenClaw](roles/web-app-openclaw/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [SeaweedFS](roles/web-app-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [File Server](roles/web-svc-file/) | beta | Static file server over HTTPS with directory listing, .well-known handling, and automatic TLS certificates | [๐Ÿ› ๏ธ](roles/web-svc-file/README.md#quick-setup) | [Corporate Design](roles/web-svc-css/), [Matomo](roles/web-app-matomo/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/) | +| [Firewall ruleset](roles/svc-net-firewall/) | beta | nftables packet filter whose rules each role owns as its own table, replaced atomically and reloaded at boot | [๐ŸŒ](https://www.netfilter.org/projects/nftables/) [๐ŸŽฌ](https://www.youtube.com/watch?v=EGKhIljDPCw) [๐Ÿ› ๏ธ](roles/svc-net-firewall/README.md#quick-setup) | | | [Flowise](roles/web-app-flowise/) | beta | Low-code visual builder for creating, testing, and publishing LLM and RAG workflows as HTTP endpoints | [๐ŸŒ](https://flowiseai.com/) [๐ŸŽฌ](https://www.youtube.com/watch?v=h_0TJTY6Ulg) [๐Ÿ› ๏ธ](roles/web-app-flowise/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [LiteLLM Gateway](roles/svc-ai-litellm/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [Ollama](roles/svc-ai-ollama/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Qdrant](roles/svc-db-qdrant/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Friendica](roles/web-app-friendica/) | beta | Decentralized, federated social-networking platform for peer-to-peer communication and community building | [๐ŸŒ](https://friendi.ca/) [๐Ÿ› ๏ธ](roles/web-app-friendica/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [MariaDB](roles/svc-db-mariadb/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [Prometheus](roles/web-app-prometheus/), [Tor Onion Service](roles/svc-net-tor/), [Universal Logout](roles/web-svc-logout/) | | [Funkwhale](roles/web-app-funkwhale/) | beta | Self-hosted, federated platform for sharing, hosting, and streaming your own music and audio library | [๐ŸŒ](https://www.funkwhale.audio/) [๐Ÿ› ๏ธ](roles/web-app-funkwhale/README.md#quick-setup) | [Backup Docker Volumes](roles/svc-bkp-volume-2-local/), [Corporate Design](roles/web-svc-css/), [Dashboard](roles/web-app-dashboard/), [Keycloak](roles/web-app-keycloak/), [Mailu](roles/web-app-mailu/), [Matomo](roles/web-app-matomo/), [OpenLDAP](roles/svc-db-openldap/), [PostgreSQL](roles/svc-db-postgres/), [Prometheus](roles/web-app-prometheus/), [Redis](roles/svc-db-redis/), [SeaweedFS](roles/web-app-seaweedfs/), [Tor Onion Service](roles/svc-net-tor/), [Typesense](roles/svc-db-typesense/), [Universal Logout](roles/web-svc-logout/) | diff --git a/docs/requirements/035-mcp-proxy-expansion.md b/docs/requirements/035-mcp-proxy-expansion.md index 55ef794b0e..9736a3cf69 100644 --- a/docs/requirements/035-mcp-proxy-expansion.md +++ b/docs/requirements/035-mcp-proxy-expansion.md @@ -488,7 +488,7 @@ A future exception requires a new requirement with a fixed operation list, dedic **`host_execution_boundary` (39):** `desk-bluray-player`, `desk-chromium`, `desk-copyq`, `desk-docker`, `desk-dotlinker`, `desk-firefox`, `desk-git`, `desk-gnome`, `desk-gnome-caffeine`, `desk-gnome-extensions`, `desk-gnome-terminal`, `desk-gnucash`, `desk-jrnl`, `desk-keepassxc`, `desk-libreoffice`, `desk-micro`, `desk-neovim`, `desk-nextcloud`, `desk-obs`, `desk-qbittorrent`, `desk-retroarch`, `desk-spotify`, `desk-ssh`, `desk-torbrowser`, `desk-virtualbox`, `desk-zoom`, `dev-arduino`, `dev-core`, `dev-java`, `dev-locales`, `dev-make`, `dev-nix`, `dev-nodejs`, `dev-python`, `drv-epson-multiprinter`, `drv-intel`, `drv-lid-switch`, `drv-non-free`, `gen-hunspell`. These roles operate a workstation, developer toolchain, device, or host package; bridging them would amount to shared shell, filesystem, browser-session, device, or host execution without an application-specific remote identity. -**`privileged_control_plane` (23):** `svc-bkp-local-2-device`, `svc-bkp-nfs-2-local`, `svc-bkp-remote-2-local`, `svc-bkp-secrets-2-local`, `svc-bkp-volume-2-local`, `svc-dns-unbound`, `svc-net-tor`, `svc-net-wireguard-core`, `svc-net-wireguard-firewalled`, `svc-net-wireguard-plain`, `svc-opt-keyboard-color`, `svc-opt-ssd-hdd`, `svc-opt-swapfile`, `svc-prx-openresty`, `svc-registry-cache`, `svc-registry-docker`, `svc-runner`, `svc-storage-nfs-client`, `svc-storage-nfs-server`, `svc-swarm-manager`, `svc-swarm-node`, `svc-virt-kata`, `update`. +**`privileged_control_plane` (24):** `svc-bkp-local-2-device`, `svc-bkp-nfs-2-local`, `svc-bkp-remote-2-local`, `svc-bkp-secrets-2-local`, `svc-bkp-volume-2-local`, `svc-dns-unbound`, `svc-net-firewall`, `svc-net-tor`, `svc-net-wireguard-core`, `svc-net-wireguard-firewalled`, `svc-net-wireguard-plain`, `svc-opt-keyboard-color`, `svc-opt-ssd-hdd`, `svc-opt-swapfile`, `svc-prx-openresty`, `svc-registry-cache`, `svc-registry-docker`, `svc-runner`, `svc-storage-nfs-client`, `svc-storage-nfs-server`, `svc-swarm-manager`, `svc-swarm-node`, `svc-virt-kata`, `update`. These roles can recover secrets, alter routing or host state, run code, change deployment state, or reach storage/control-plane sockets; they require a separate audited operations gateway and human approval rather than general-purpose application MCP. **`shared_engine_isolation` (4):** `svc-db-memcached`, `svc-db-openldap`, `svc-db-postgres`, `svc-db-rabbitmq`. diff --git a/roles/svc-net-firewall/README.md b/roles/svc-net-firewall/README.md new file mode 100644 index 0000000000..a6ef462a55 --- /dev/null +++ b/roles/svc-net-firewall/README.md @@ -0,0 +1,90 @@ +# Firewall ruleset + +## Description + +[nftables](https://www.netfilter.org/projects/nftables/) is the packet filtering framework of the Linux kernel and the successor to `iptables`. Its rules live in memory: a reboot drops them unless something loads them again. + +## Overview + +This role gives every other role one place to put packet filter rules. A role that needs them writes `templates/nftables.conf.j2` declaring its own nftables table and includes this role; the fragment is rendered to `/etc/nftables.d/.conf`, checked, loaded, and read back, so a deploy fails rather than quietly leaving a gap. Deployed on its own, the role installs nftables, the ruleset that loads every fragment, and the `nftables.service` that replays them at boot. + +One table per role is what makes the rules composable. A fragment replaces its own table and touches no other, so two roles on one host neither overwrite nor merge into each other, and a broken fragment fails alone instead of freezing everyone's rules. + +Replacing beats adding. `nft -f` on an existing table merges, so a rule dropped from the declaration would stay in the kernel and the ruleset could only ever grow. Each fragment therefore creates, deletes and redeclares its table in one transaction, which is also why the table is never observed empty: unlike a flush followed by rules, there is no window in which the boundary is gone. + +## Cosmos + +The diagram places Firewall ruleset in the Infinito.Nexus cosmos: the components it deploys (capabilities), the central services it consumes (dependencies), and its outward reach (federation and bridged external networks). + +```mermaid +flowchart LR + subgraph role [svc-net-firewall ๐Ÿ’ป] + svc_firewall["firewall"] + end +``` + +Solid `1:1` edges are fixed relationships; dashed `0..1` edges are conditional (enabled only in matching deployments); red `0..0` edges are turned off in this role. Node markers show the role's deploy modes (๐Ÿ’ป host, ๐Ÿณ compose, ๐Ÿ swarm); โŒ marks a service that is explicitly turned off, and โš™๏ธ an Ansible role dependency declared in `meta/main.yml`. + +## Features + +- **Owned per role:** Each consumer declares one table under its own name, so rules are added and retired with the role that needs them. +- **Atomic:** A fragment is applied in a single `nft -f` transaction, so the table is never observed half built. +- **Persistent:** `nftables.service` loads every fragment at boot, so rules outlive a reboot without a unit of this project's own. The ruleset it reads is a distribution decision, so the path follows `os_family`: RedHat reads `/etc/sysconfig/nftables.conf`, the others `/etc/nftables.conf`. +- **Independent of docker:** Docker owns its own tables and never touches these, so no rule has to be inserted into a chain another component controls. The unit is only enabled, never started or restarted: on Debian its `ExecStop` flushes the entire ruleset, docker's tables included, and the fragments are loaded explicitly anyway. +- **Verified:** `nft -c` rejects a fragment before it is written, and the loaded table is read back and refused when it carries no chain. A table that exists is not a rule that runs, so the policy itself is proven by the role that owns it. + +## Use Cases + +- Bounding a service port that carries no access policy of its own, where the packet filter is the only boundary available. +- Holding a forwarding or masquerade rule that a tunnel needs, without losing it at the next reboot. + +## Quick Setup + +### Development + +Clone, set up the workstation, and deploy Firewall ruleset onto the local stack: + +```bash +git clone https://github.com/infinito-nexus/core.git +cd core +make onboard +make compose-deploy mode=reinstall apps=svc-net-firewall full_cycle=false +``` + +### Production + +Install Firewall ruleset directly onto the target machine: clone the repository, install the OS prerequisites and the repository toolchain, then deploy against localhost over a local connection (no SSH, no container): + +```bash +git clone https://github.com/infinito-nexus/core.git +cd core +bash scripts/install/package.sh +make install +source scripts/meta/env/load.sh + +APP=svc-net-firewall +DOMAIN= +TLS_MODE=self_signed +SSH_PUBLIC_KEY="" +INVENTORY=inventories/production +infinito administration inventory provision "$INVENTORY" \ + --inventory-file "$INVENTORY/devices.yml" \ + --host localhost \ + --include "$APP" \ + --vars "{\"TLS_MODE\": \"$TLS_MODE\", \"DOMAIN_PRIMARY\": \"$DOMAIN\", \"users\": {\"administrator\": {\"authorized_keys\": [\"$SSH_PUBLIC_KEY\"]}}}" +infinito administration deploy dedicated "$INVENTORY/devices.yml" \ + --password-file "$INVENTORY/.password" \ + --diff -vv +``` + +## Further Resources + +- [nftables Wiki](https://wiki.nftables.org/) +- [nft(8) manual page](https://www.netfilter.org/projects/nftables/manpage.html) +- [Moving from iptables to nftables](https://wiki.nftables.org/wiki-nftables/index.php/Moving_from_iptables_to_nftables) + +## Credits + +Implemented by **[Kevin Veen-Birkenbach](https://www.veen.world)**. +Part of the [Infinito.Nexus Project](https://s.infinito.nexus/code) and maintained by [Kevin Veen-Birkenbach](https://www.veen.world). +Licensed under the [Infinito.Nexus Community License (Non-Commercial)](https://s.infinito.nexus/license). diff --git a/roles/svc-net-firewall/meta/info.yml b/roles/svc-net-firewall/meta/info.yml new file mode 100644 index 0000000000..ca99331c28 --- /dev/null +++ b/roles/svc-net-firewall/meta/info.yml @@ -0,0 +1,5 @@ +--- +logo: + class: fa-solid fa-shield-halved +homepage: https://www.netfilter.org/projects/nftables/ +video: https://www.youtube.com/watch?v=EGKhIljDPCw diff --git a/roles/svc-net-firewall/meta/main.yml b/roles/svc-net-firewall/meta/main.yml new file mode 100644 index 0000000000..93ca3d7d86 --- /dev/null +++ b/roles/svc-net-firewall/meta/main.yml @@ -0,0 +1,27 @@ +--- +galaxy_info: + author: Kevin Veen-Birkenbach + description: nftables packet filter whose rules each role owns as its own table, replaced atomically and reloaded at boot + company: | + Kevin Veen-Birkenbach + https://www.veen.world + license: Infinito.Nexus Community License (Non-Commercial) + min_ansible_version: "2.9" + galaxy_tags: + - firewall + - nftables + - netfilter + - security + issue_tracker_url: https://s.infinito.nexus/issues + platforms: + - name: ArchLinux + versions: [all] + - name: Debian + versions: [all] + - name: EL + versions: [all] + - name: Fedora + versions: [all] + - name: Ubuntu + versions: [all] +dependencies: [] diff --git a/roles/svc-net-firewall/meta/packages.yml b/roles/svc-net-firewall/meta/packages.yml new file mode 100644 index 0000000000..06709ff242 --- /dev/null +++ b/roles/svc-net-firewall/meta/packages.yml @@ -0,0 +1,8 @@ +--- +nftables: + Archlinux: + - nftables + Debian: + - nftables + RedHat: + - nftables diff --git a/roles/svc-net-firewall/meta/services.yml b/roles/svc-net-firewall/meta/services.yml new file mode 100644 index 0000000000..15c0b390d5 --- /dev/null +++ b/roles/svc-net-firewall/meta/services.yml @@ -0,0 +1,16 @@ +--- +# nocheck: playwright-service-flag host packet filter, no container and no web surface +firewall: + modes: + host: + enabled: true + compose: + enabled: false + swarm: + enabled: false + lifecycle: beta + min_storage: "50MB" + cpus: "0.1" + mem_reservation: "32m" + mem_limit: "64m" + pids_limit: 64 diff --git a/roles/svc-net-firewall/tasks/01_host.yml b/roles/svc-net-firewall/tasks/01_host.yml new file mode 100644 index 0000000000..9474747829 --- /dev/null +++ b/roles/svc-net-firewall/tasks/01_host.yml @@ -0,0 +1,32 @@ +--- +- name: "๐Ÿ“ฆ Firewall | Install nftables" + become: true + package_install: + id: nftables + +- name: "๐Ÿ“ Firewall | Ensure the fragment directory exists" + become: true + ansible.builtin.file: + path: "{{ FIREWALL_FRAGMENT_DIR }}" + state: directory + owner: root + group: root + mode: "0750" + +- name: "๐Ÿงฑ Firewall | Install the ruleset every fragment is loaded from" + become: true + ansible.builtin.template: # nocheck: stack-host-template per-host packet filter: every host carrying rules needs the ruleset, not only the stack host + src: main.nft.j2 + dest: "{{ FIREWALL_MAIN_CONF }}" + owner: root + group: root + mode: "0640" + backup: true + +- name: "๐Ÿš€ Firewall | Arm the ruleset for boot without touching the live one" + become: true + ansible.builtin.systemd_service: + name: "{{ FIREWALL_SERVICE }}" + enabled: true + +- include_tasks: utils/once/flag.yml diff --git a/roles/svc-net-firewall/tasks/02_fragment.yml b/roles/svc-net-firewall/tasks/02_fragment.yml new file mode 100644 index 0000000000..5ed9c5900e --- /dev/null +++ b/roles/svc-net-firewall/tasks/02_fragment.yml @@ -0,0 +1,43 @@ +--- +- name: "๐Ÿ” Firewall | Resolve the fragment of '{{ FIREWALL_FRAGMENT_ROLE }}'" + ansible.builtin.set_fact: + firewall_fragment_src: >- + {{ lookup('first_found', + { + 'files': [ FIREWALL_FRAGMENT_SRC ], + 'paths': [ lookup('path_absolute', 'roles', FIREWALL_FRAGMENT_ROLE) ] + } + ) }} + firewall_loaded_table: "{{ FIREWALL_TABLE }}" + +- name: "๐Ÿงฑ Firewall | Stage the ruleset of '{{ FIREWALL_FRAGMENT_ROLE }}'" + become: true + ansible.builtin.template: # nocheck: stack-host-template per-host packet filter: every host carrying the rules needs the fragment, not only the stack host + src: "{{ firewall_fragment_src }}" + dest: "{{ FIREWALL_FRAGMENT_FILE }}" + owner: root + group: root + mode: "0640" + validate: "nft -c -f %s" + +- name: "๐Ÿš€ Firewall | Load the ruleset of '{{ FIREWALL_FRAGMENT_ROLE }}'" + become: true + ansible.builtin.command: + argv: [nft, -f, "{{ FIREWALL_FRAGMENT_FILE }}"] + changed_when: true + +- name: "๐Ÿ” Firewall | Read back the table of '{{ FIREWALL_FRAGMENT_ROLE }}'" + become: true + ansible.builtin.command: + argv: [nft, list, table, inet, "{{ FIREWALL_TABLE }}"] + register: firewall_loaded_ruleset + changed_when: false + +- name: "โœ… Firewall | Refuse a table that carries no chain" + ansible.builtin.assert: + that: + - "'chain ' in firewall_loaded_ruleset.stdout" + fail_msg: >- + {{ FIREWALL_FRAGMENT_ROLE }} loaded {{ FIREWALL_TABLE }} but the table + holds no chain, so nothing it declares is hooked into the packet path. + A table that exists is not a rule that runs. diff --git a/roles/svc-net-firewall/tasks/main.yml b/roles/svc-net-firewall/tasks/main.yml new file mode 100644 index 0000000000..435c971317 --- /dev/null +++ b/roles/svc-net-firewall/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- name: "๐Ÿงฑ Firewall | Prepare the host ruleset" + ansible.builtin.include_tasks: 01_host.yml + when: run_once_svc_net_firewall is not defined + +- name: "๐Ÿงฑ Firewall | Apply the ruleset of '{{ FIREWALL_FRAGMENT_ROLE }}'" + ansible.builtin.include_tasks: 02_fragment.yml + when: FIREWALL_FRAGMENT_ROLE | length > 0 diff --git a/roles/svc-net-firewall/templates/main.nft.j2 b/roles/svc-net-firewall/templates/main.nft.j2 new file mode 100644 index 0000000000..466e80c23c --- /dev/null +++ b/roles/svc-net-firewall/templates/main.nft.j2 @@ -0,0 +1,3 @@ +#!/usr/sbin/nft -f +# Managed by Infinito.Nexus (roles/svc-net-firewall). Do not edit by hand. +include "{{ FIREWALL_FRAGMENT_DIR }}/*.conf" diff --git a/roles/svc-net-firewall/vars/main.yml b/roles/svc-net-firewall/vars/main.yml new file mode 100644 index 0000000000..0303fda6d6 --- /dev/null +++ b/roles/svc-net-firewall/vars/main.yml @@ -0,0 +1,15 @@ +--- +application_id: svc-net-firewall + +FIREWALL_MAIN_CONF_BY_OS_FAMILY: + Archlinux: "/etc/nftables.conf" + Debian: "/etc/nftables.conf" + RedHat: "/etc/sysconfig/nftables.conf" + +FIREWALL_MAIN_CONF: "{{ FIREWALL_MAIN_CONF_BY_OS_FAMILY[ansible_facts['os_family']] }}" +FIREWALL_FRAGMENT_DIR: "/etc/nftables.d" +FIREWALL_SERVICE: "nftables.service" +FIREWALL_FRAGMENT_SRC: "templates/nftables.conf.j2" +FIREWALL_FRAGMENT_ROLE: "{{ ansible_parent_role_names | default([]) | first | default('') }}" +FIREWALL_FRAGMENT_FILE: "{{ [ FIREWALL_FRAGMENT_DIR, FIREWALL_FRAGMENT_ROLE ~ '.conf' ] | path_join }}" +FIREWALL_TABLE: "infinito_{{ FIREWALL_FRAGMENT_ROLE | regex_replace('[^a-zA-Z0-9]', '_') }}" diff --git a/roles/svc-net-tor/files/test/onion_ports.py b/roles/svc-net-tor/files/test/onion_ports.py index b1985311ae..a372b65d47 100755 --- a/roles/svc-net-tor/files/test/onion_ports.py +++ b/roles/svc-net-tor/files/test/onion_ports.py @@ -61,15 +61,11 @@ def _required(name: str, hint: str) -> str: return value -def connect_through_socks( - proxy: str, host: str, port: int, timeout: float -) -> None: +def connect_through_socks(proxy: str, host: str, port: int, timeout: float) -> None: """Open a SOCKS5 CONNECT to ``host:port`` and raise when it is refused.""" proxy_host, _, proxy_port = proxy.rpartition(":") try: - sock = socket.create_connection( - (proxy_host, int(proxy_port)), timeout=timeout - ) + sock = socket.create_connection((proxy_host, int(proxy_port)), timeout=timeout) except OSError as error: raise ProbeError(f"socks proxy unreachable: {error}") from None diff --git a/roles/svc-net-tor/handlers/main.yml b/roles/svc-net-tor/handlers/main.yml deleted file mode 100644 index 7f1b99b5a5..0000000000 --- a/roles/svc-net-tor/handlers/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: "๐Ÿ›ก๏ธ Apply the Tor egress guard" - listen: apply tor egress guard - ansible.builtin.systemd_service: - name: "{{ TOR_EGRESS_GUARD_UNIT | basename }}" - state: restarted - daemon_reload: true - become: true diff --git a/roles/svc-net-tor/lookup_plugins/tor_egress_clients.py b/roles/svc-net-tor/lookup_plugins/tor_egress_clients.py index 51f5d48665..bd0a9647ae 100644 --- a/roles/svc-net-tor/lookup_plugins/tor_egress_clients.py +++ b/roles/svc-net-tor/lookup_plugins/tor_egress_clients.py @@ -1,21 +1,10 @@ -"""Lookup `tor_egress_clients`: who may use this node's Tor egress ports. +"""Lookup `tor_egress_clients`: the address space Tor's egress ports admit. -Tor's DNSPort and TransPort carry no access policy of their own, unlike -SocksPort with its SocksPolicy, and a swarm-published port cannot be bound to a -single host address. The filter chain installed by ``tasks/router.yml`` is -therefore what bounds them, and this lookup is the list it admits. - -Deliberately not the three RFC1918 blocks: ``10.0.0.0/8`` would hand the -operator's whole LAN a resolver and a transparent proxy. Admitted is exactly the -address space this deployment declares for itself: - -* loopback, for the node's own dnsmasq, -* every base of ``NETWORK_DOCKER_ADDRESS_POOLS``, -* ``NETWORK_SWARM_GWBRIDGE_POOL``, the gateway bridge docker creates outside - those pools and without which overlay tasks lose their route out, -* ``NETWORK_ROLE_SUBNET_POOL`` and the blocks of the roles named in - ``NETWORK_ROLE_SUBNET_EXCEPTION_ROLES``, resolved through ``role_subnets`` so - the CIDR stays declared in the role that owns it. +Deliberately not the three RFC1918 blocks: 10.0.0.0/8 would hand the operator's +whole LAN a resolver and a transparent proxy that carry no access policy of +their own. What is admitted is what this deployment declares for itself, in rule +order: loopback, docker's address pools, the swarm gateway bridge, the role +subnet pool, and the subnets of the roles that sit outside it. Usage: @@ -29,6 +18,7 @@ from typing import Any from ansible.errors import AnsibleError from ansible.plugins.loader import lookup_loader from ansible.plugins.lookup import LookupBase +from ansible.template import trust_as_template _REQUIRED = ( "NETWORK_LOOPBACK_CIDR", @@ -39,20 +29,21 @@ _REQUIRED = ( ) -def _pool_bases(pools: Any) -> list[str]: - if not isinstance(pools, (list, tuple)): - raise AnsibleError( - "tor_egress_clients: NETWORK_DOCKER_ADDRESS_POOLS must be a list" - ) - bases: list[str] = [] - for pool in pools: - base = pool.get("base") if isinstance(pool, dict) else None - if not base: - raise AnsibleError( - f"tor_egress_clients: address pool {pool!r} declares no base" - ) - bases.append(str(base).strip()) - return bases +def _rendered(templar: Any, value: Any) -> str: + """A variable's value with its Jinja resolved. + + Group vars reach a lookup as whatever the file holds, and several of these + are references rather than literals. Passing one through unrendered would + put the template text itself into a source address, where it cannot match + and therefore silently narrows the admitted set. + """ + text = "" if value is None else str(value).strip() + if "{{" not in text or templar is None: + return text + rendered = str(templar.template(trust_as_template(text))).strip() + if "{{" in rendered: + raise AnsibleError(f"tor_egress_clients: {text!r} did not render to an address") + return rendered class LookupModule(LookupBase): @@ -68,28 +59,43 @@ class LookupModule(LookupBase): missing = [name for name in _REQUIRED if variables.get(name) is None] if missing: raise AnsibleError( - "tor_egress_clients: missing group variable(s) " - f"{', '.join(missing)}; the guard must never be narrowed by a " - "value that silently defaulted" + "tor_egress_clients: missing variable(s) " + f"{', '.join(missing)}; an incomplete list would install a guard " + "that admits more than it was asked to" ) + templar = getattr(self, "_templar", None) excepted = lookup_loader.get( "role_subnets", loader=getattr(self, "_loader", None), - templar=getattr(self, "_templar", None), + templar=templar, ).run( [variables["NETWORK_ROLE_SUBNET_EXCEPTION_ROLES"]], variables=variables, )[0] - cidrs = [ - str(variables["NETWORK_LOOPBACK_CIDR"]).strip(), - *_pool_bases(variables["NETWORK_DOCKER_ADDRESS_POOLS"]), - str(variables["NETWORK_SWARM_GWBRIDGE_POOL"]).strip(), - str(variables["NETWORK_ROLE_SUBNET_POOL"]).strip(), - *excepted, + pools = variables["NETWORK_DOCKER_ADDRESS_POOLS"] + if not isinstance(pools, (list, tuple)): + raise AnsibleError( + "tor_egress_clients: NETWORK_DOCKER_ADDRESS_POOLS must be a list" + ) + bases = [] + for pool in pools: + base = pool.get("base") if isinstance(pool, dict) else None + if not base: + raise AnsibleError( + f"tor_egress_clients: address pool {pool!r} declares no base" + ) + bases.append(_rendered(templar, base)) + + ordered = [ + _rendered(templar, variables["NETWORK_LOOPBACK_CIDR"]), + *bases, + _rendered(templar, variables["NETWORK_SWARM_GWBRIDGE_POOL"]), + _rendered(templar, variables["NETWORK_ROLE_SUBNET_POOL"]), + *[_rendered(templar, subnet) for subnet in excepted], ] seen: dict[str, None] = {} - for cidr in cidrs: + for cidr in ordered: seen.setdefault(cidr, None) return [list(seen)] diff --git a/roles/svc-net-tor/tasks/router.yml b/roles/svc-net-tor/tasks/router.yml index 51a2fcbe14..7d400f9521 100644 --- a/roles/svc-net-tor/tasks/router.yml +++ b/roles/svc-net-tor/tasks/router.yml @@ -88,101 +88,22 @@ regexp: "^(hosts:.*\\bresolve)\\s*\\[!UNAVAIL=return\\](.*)$" line: "\\1 [SUCCESS=return NOTFOUND=continue TRYAGAIN=return]\\2" backrefs: true -- name: "๐Ÿ”€ Egress router | Redirect .onion virtual-IP TCP to Tor TransPort" - ansible.builtin.iptables: - table: nat - chain: OUTPUT - protocol: tcp - destination: "{{ TOR_EGRESS_VIRTUAL_NET_IPV4 }}" - jump: REDIRECT - to_ports: "{{ TOR_TRANS_PORT | string }}" - comment: "Tor egress router: .onion virtual range -> TransPort" +- name: "๐Ÿ”€ Egress router | Route .onion traffic and bound the ports that carry no policy" + ansible.builtin.include_role: + name: svc-net-firewall -- name: "๐Ÿ”€ Egress router | Send container .onion traffic to the bridge TransPort" - when: TOR_CONTAINER_DNS_HOST | length > 0 - ansible.builtin.iptables: - table: nat - chain: PREROUTING - protocol: tcp - destination: "{{ TOR_EGRESS_VIRTUAL_NET_IPV4 }}" - jump: DNAT - to_destination: "{{ TOR_CONTAINER_DNS_HOST }}:{{ TOR_TRANS_PORT }}" - comment: "Tor egress router: container .onion virtual range -> TransPort" - -- name: "๐Ÿ”€ Egress router | Prove the container redirect is installed" - when: TOR_CONTAINER_DNS_HOST | length > 0 +- name: "๐Ÿ›ก๏ธ Egress guard | Read back the chain that bounds the ports" + become: true ansible.builtin.command: - argv: - - iptables - - -t - - nat - - -C - - PREROUTING - - -p - - tcp - - -d - - "{{ TOR_EGRESS_VIRTUAL_NET_IPV4 }}" - - -j - - DNAT - - --to-destination - - "{{ TOR_CONTAINER_DNS_HOST }}:{{ TOR_TRANS_PORT }}" + argv: [nft, list, chain, inet, "{{ firewall_loaded_table }}", guard] + register: tor_egress_guard_chain changed_when: false -- name: "๐Ÿ›ก๏ธ Egress guard | Stage the guard applier" - when: DEPLOYMENT_MODE == 'swarm' - ansible.builtin.template: - src: tor-egress-guard.sh.j2 - dest: "{{ TOR_EGRESS_GUARD_SCRIPT }}" - owner: root - group: root - mode: "0750" - notify: apply tor egress guard - -- name: "๐Ÿ›ก๏ธ Egress guard | Install the unit that reapplies it after docker" - when: DEPLOYMENT_MODE == 'swarm' - ansible.builtin.template: - src: tor-egress-guard.service.j2 - dest: "{{ TOR_EGRESS_GUARD_UNIT }}" - owner: root - group: root - mode: "0644" - notify: apply tor egress guard - -- name: "๐Ÿ›ก๏ธ Egress guard | Enable the guard across reboots" - when: DEPLOYMENT_MODE == 'swarm' - ansible.builtin.systemd_service: - name: "{{ TOR_EGRESS_GUARD_UNIT | basename }}" - enabled: true - state: started - daemon_reload: true - -- name: "๐Ÿ›ก๏ธ Egress guard | Flush handlers so the guard is live before checking" - when: DEPLOYMENT_MODE == 'swarm' - ansible.builtin.meta: flush_handlers - -- name: "๐Ÿ›ก๏ธ Egress guard | Prove the guard rejects everything it must" - when: DEPLOYMENT_MODE == 'swarm' - ansible.builtin.command: - argv: - - iptables - - -C - - "{{ TOR_EGRESS_GUARD_CHAIN }}" - - -j - - DROP - changed_when: false - -- name: "๐Ÿ›ก๏ธ Egress guard | Prove every guarded port reaches the guard" - when: DEPLOYMENT_MODE == 'swarm' - ansible.builtin.command: - argv: - - iptables - - -C - - "{{ item.0 }}" - - -p - - "{{ item.1.protocol }}" - - --dport - - "{{ item.1.port | string }}" - - -j - - "{{ TOR_EGRESS_GUARD_CHAIN }}" - changed_when: false - loop: "{{ TOR_EGRESS_GUARD_HOOK_CHAINS | product(TOR_EGRESS_GUARDED_PORTS) | list }}" +- name: "๐Ÿ›ก๏ธ Egress guard | Refuse a guard that drops nothing" + ansible.builtin.assert: + that: + - "'drop' in tor_egress_guard_chain.stdout" + fail_msg: >- + The guard chain in {{ firewall_loaded_table }} carries no drop, so + TransPort and DNSPort are reachable from every source. These two ports + have no access policy of their own; this chain is the only boundary. diff --git a/roles/svc-net-tor/templates/nftables.conf.j2 b/roles/svc-net-tor/templates/nftables.conf.j2 new file mode 100644 index 0000000000..69155b8a43 --- /dev/null +++ b/roles/svc-net-tor/templates/nftables.conf.j2 @@ -0,0 +1,38 @@ +{# Loaded by roles/svc-net-firewall into {{ FIREWALL_TABLE }}. #} +table inet {{ FIREWALL_TABLE }} +delete table inet {{ FIREWALL_TABLE }} +table inet {{ FIREWALL_TABLE }} { + set admitted { + type ipv4_addr + flags interval + elements = { {{ TOR_EGRESS_CLIENT_CIDRS | join(', ') }} } + } + + chain guard { +{% for entry in TOR_EGRESS_GUARDED_PORTS %} + ip saddr != @admitted {{ entry.protocol }} dport {{ entry.port }} drop +{% endfor %} + } + + chain input { + type filter hook input priority -10; policy accept; + jump guard + } + + chain forward { + type filter hook forward priority -10; policy accept; + jump guard + } + + chain output { + type nat hook output priority -150; policy accept; + ip daddr {{ TOR_EGRESS_VIRTUAL_NET_IPV4 }} meta l4proto tcp redirect to :{{ TOR_TRANS_PORT }} + } +{% if TOR_CONTAINER_DNS_HOST | length > 0 %} + + chain prerouting { + type nat hook prerouting priority -150; policy accept; + ip daddr {{ TOR_EGRESS_VIRTUAL_NET_IPV4 }} meta l4proto tcp dnat ip to {{ TOR_CONTAINER_DNS_HOST }}:{{ TOR_TRANS_PORT }} + } +{% endif %} +} diff --git a/roles/svc-net-tor/templates/tor-egress-guard.service.j2 b/roles/svc-net-tor/templates/tor-egress-guard.service.j2 deleted file mode 100644 index fae8f0cdc6..0000000000 --- a/roles/svc-net-tor/templates/tor-egress-guard.service.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Infinito.Nexus Tor egress guard -Documentation=file://{{ TOR_EGRESS_GUARD_SCRIPT }} -After=docker.service -Requires=docker.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart={{ TOR_EGRESS_GUARD_SCRIPT }} - -[Install] -WantedBy=multi-user.target diff --git a/roles/svc-net-tor/templates/tor-egress-guard.sh.j2 b/roles/svc-net-tor/templates/tor-egress-guard.sh.j2 deleted file mode 100644 index c8892062a4..0000000000 --- a/roles/svc-net-tor/templates/tor-egress-guard.sh.j2 +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Managed by Infinito.Nexus (roles/svc-net-tor) โ€” do not edit by hand. -set -euo pipefail - -CHAIN={{ TOR_EGRESS_GUARD_CHAIN | quote }} - -iptables -N "${CHAIN}" 2>/dev/null || true -iptables -F "${CHAIN}" - -{% for cidr in TOR_EGRESS_CLIENT_CIDRS %} -iptables -A "${CHAIN}" -s {{ cidr | quote }} -j ACCEPT -{% endfor %} -iptables -A "${CHAIN}" -j DROP - -{% for chain in TOR_EGRESS_GUARD_HOOK_CHAINS %} -iptables -N {{ chain | quote }} 2>/dev/null || true -{% for entry in TOR_EGRESS_GUARDED_PORTS %} -iptables -C {{ chain | quote }} -p {{ entry.protocol | quote }} --dport {{ entry.port | quote }} -j "${CHAIN}" 2>/dev/null \ - || iptables -I {{ chain | quote }} -p {{ entry.protocol | quote }} --dport {{ entry.port | quote }} -j "${CHAIN}" -{% endfor %} -{% endfor %} diff --git a/roles/svc-net-tor/vars/main.yml b/roles/svc-net-tor/vars/main.yml index 9f0c392b3f..24ae0ebfa5 100644 --- a/roles/svc-net-tor/vars/main.yml +++ b/roles/svc-net-tor/vars/main.yml @@ -13,13 +13,6 @@ TOR_TRANS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.t TOR_DNS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.dns') }}" TOR_DNSMASQ_CONF: "/etc/dnsmasq.d/tor-onion.conf" -TOR_EGRESS_GUARD_CHAIN: INFINITO_TOR_EGRESS -TOR_EGRESS_GUARD_SCRIPT: "/usr/local/bin/tor-egress-guard.sh" -TOR_EGRESS_GUARD_UNIT: "/etc/systemd/system/tor-egress-guard.service" -TOR_EGRESS_GUARD_HOOK_CHAINS: - - INPUT - - DOCKER-USER -TOR_EGRESS_CLIENT_CIDRS: "{{ lookup('tor_egress_clients') }}" TOR_EGRESS_GUARDED_PORTS: - port: "{{ TOR_TRANS_PORT }}" protocol: tcp @@ -27,6 +20,7 @@ TOR_EGRESS_GUARDED_PORTS: protocol: tcp - port: "{{ TOR_DNS_PORT }}" protocol: udp +TOR_EGRESS_CLIENT_CIDRS: "{{ lookup('tor_egress_clients') }}" TOR_DNSMASQ_DROPIN: "/etc/systemd/system/dnsmasq.service.d/infinito-restart.conf" # nocheck: host-local-fact meta/services.yml pins this role to the manager, so the fact describes the node the hidden service forwards on TOR_CONTAINER_DNS_HOST: "{{ ansible_facts['docker0']['ipv4']['address'] | default('', true) }}" diff --git a/tests/integration/ansible/jinja/test_variable_definitions.py b/tests/integration/ansible/jinja/test_variable_definitions.py index 605fd2ebd5..0092ae1603 100644 --- a/tests/integration/ansible/jinja/test_variable_definitions.py +++ b/tests/integration/ansible/jinja/test_variable_definitions.py @@ -246,6 +246,7 @@ class TestVariableDefinitions(unittest.TestCase): "ansible_memtotal_mb", "omit", "group_names", + "ansible_parent_role_names", "ansible_processor_vcpus", "ansible_service_mgr", "ansible_distribution_release", @@ -293,6 +294,7 @@ class TestVariableDefinitions(unittest.TestCase): "ansible_memtotal_mb", "omit", "group_names", + "ansible_parent_role_names", "ansible_processor_vcpus", "ansible_distribution_release", "ansible_distribution", diff --git a/tests/lint/ansible/jinja/test_shell_templates_are_valid.py b/tests/lint/ansible/jinja/test_shell_templates_are_valid.py index 7df56911dc..921e159c9e 100644 --- a/tests/lint/ansible/jinja/test_shell_templates_are_valid.py +++ b/tests/lint/ansible/jinja/test_shell_templates_are_valid.py @@ -77,15 +77,7 @@ class _Placeholder(jinja2.Undefined): return self -_SAMPLE = { - "TOR_EGRESS_GUARD_CHAIN": "INFINITO_TOR_EGRESS", - "TOR_EGRESS_CLIENT_CIDRS": ["127.0.0.0/8", "10.208.0.0/12"], - "TOR_EGRESS_GUARD_HOOK_CHAINS": ["INPUT", "DOCKER-USER"], - "TOR_EGRESS_GUARDED_PORTS": [ - {"port": "9040", "protocol": "tcp"}, - {"port": "9053", "protocol": "udp"}, - ], -} +_SAMPLE: dict[str, object] = {} def _shell_templates() -> list[Path]: diff --git a/tests/unit/python/roles/svc-net-firewall/__init__.py b/tests/unit/python/roles/svc-net-firewall/__init__.py new file mode 100644 index 0000000000..8ee16d7567 --- /dev/null +++ b/tests/unit/python/roles/svc-net-firewall/__init__.py @@ -0,0 +1,3 @@ +from pathlib import Path + +PROJECT_ROOT: Path = Path(__file__).resolve().parents[5] diff --git a/tests/unit/python/roles/svc-net-firewall/test_fragments.py b/tests/unit/python/roles/svc-net-firewall/test_fragments.py new file mode 100644 index 0000000000..fc1ce828ee --- /dev/null +++ b/tests/unit/python/roles/svc-net-firewall/test_fragments.py @@ -0,0 +1,208 @@ +"""Every role's nftables fragment parses, and replaces its table instead of merging. + +``nft -f`` on ``table inet x { ... }`` MERGES into an existing table: a rule +dropped from the declaration stays in the kernel, and the ruleset can only ever +grow. The three-line idiom below is what makes a fragment a replacement: + + table inet x <- creates it when absent, so the delete cannot fail + delete table inet x <- removes whatever the last deploy left + table inet x { ... } <- the declaration, applied in the same transaction + +``nft -f`` is atomic, so the table is never observed empty. Without the first +line the delete fails on a fresh host; without the second the fragment merges. + +The checks run the real ``nft`` over the rendered text, so a fragment that is +valid Jinja but invalid nftables fails here rather than on the host. They render +every deploy mode, because a fragment may branch on one, and feed plausible +rather than real values: nft needs shapes, not this host's addresses. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import tempfile +import unittest +from contextlib import contextmanager +from functools import lru_cache +from pathlib import Path + +import jinja2 + +from utils.cache.files import iter_project_files, read_text + +from . import PROJECT_ROOT + +FRAGMENT = "templates/nftables.conf.j2" +TABLE = "infinito_probe" + +COMMON = { + "FIREWALL_TABLE": TABLE, + "TOR_EGRESS_CLIENT_CIDRS": ["127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], + "TOR_EGRESS_GUARDED_PORTS": [ + {"port": "9040", "protocol": "tcp"}, + {"port": "9053", "protocol": "tcp"}, + {"port": "9053", "protocol": "udp"}, + ], + "WIREGUARD_CLIENT_INTERFACE": "wg0-client", + "WIREGUARD_UPLINK_INTERFACE": "eth0", + "TOR_EGRESS_VIRTUAL_NET_IPV4": "10.192.0.0/10", + "TOR_TRANS_PORT": "9040", + "TOR_CONTAINER_DNS_HOST": "172.17.0.1", +} + +MODES = ("compose", "swarm") + + +def _run(argv: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(argv, capture_output=True, text=True, check=False) + + +@lru_cache(maxsize=1) +def _nft_can_check() -> bool: + """Whether `nft -c` works here. + + It is not a parser: it opens netlink and builds a cache against the live + kernel first, so on a runner without CAP_NET_ADMIN it fails on a valid + ruleset. Probing the binary's presence would make this suite red there. + """ + if shutil.which("nft") is None: + return False + with tempfile.NamedTemporaryFile("w", suffix=".conf") as handle: + handle.write("table inet infinito_capability_probe\n") + handle.flush() + return _run(["nft", "-c", "-f", handle.name]).returncode == 0 + + +@contextmanager +def _network_namespace(name: str): + """An empty namespace, so a loaded ruleset cannot reach the live one.""" + created = _run(["ip", "netns", "add", name]) + if created.returncode != 0: + raise unittest.SkipTest(f"cannot create a network namespace: {created.stderr}") + try: + yield ["ip", "netns", "exec", name] + finally: + _run(["ip", "netns", "del", name]) + + +def _fragments() -> list[str]: + suffix = "/" + FRAGMENT + return sorted( + path + for path in iter_project_files(extensions=(".j2",), exclude_tests=True) + if path.endswith(suffix) + ) + + +def _render(path: str, mode: str = "swarm") -> str: + environment = jinja2.Environment( + keep_trailing_newline=True, + trim_blocks=True, + autoescape=False, # noqa: S701 nftables, not markup + ) + return environment.from_string(read_text(path)).render( + DEPLOYMENT_MODE=mode, **COMMON + ) + + +class TestFirewallFragments(unittest.TestCase): + def test_at_least_one_fragment_exists(self) -> None: + """Otherwise every assertion below passes over an empty list.""" + self.assertTrue(_fragments(), "no role declares templates/nftables.conf.j2") + + def test_each_fragment_replaces_its_table(self) -> None: + for path in _fragments(): + rel = Path(path).relative_to(PROJECT_ROOT).as_posix() + with self.subTest(fragment=rel): + lines = [ + line.strip() for line in _render(path).splitlines() if line.strip() + ] + declarations = [ + line for line in lines if re.match(r"^(delete )?table inet ", line) + ] + self.assertEqual( + declarations[:3], + [ + f"table inet {TABLE}", + f"delete table inet {TABLE}", + f"table inet {TABLE} {{", + ], + f"{rel} must open with the create, delete and declare idiom, " + "or nft merges the new rules into the old table", + ) + + def test_each_fragment_parses(self) -> None: + if not _nft_can_check(): + self.skipTest("nft -c needs CAP_NET_ADMIN, which this runner lacks") + nft = shutil.which("nft") + for path in _fragments(): + rel = Path(path).relative_to(PROJECT_ROOT).as_posix() + for mode in MODES: + with self.subTest(fragment=rel, mode=mode): + with tempfile.NamedTemporaryFile("w", suffix=".conf") as handle: + handle.write(_render(path, mode)) + handle.flush() + result = subprocess.run( + [nft, "-c", "-f", handle.name], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual( + result.returncode, + 0, + f"{rel} is not valid nftables in {mode}:\n{result.stderr}", + ) + + def test_each_fragment_loads_and_reloads(self) -> None: + """A dry run cannot show what happens on the second deploy. + + Loading the same fragment twice is what proves the idiom: the second + pass meets the table the first one left and must replace it, which is + exactly the case ``nft -c`` never sees. + """ + if not _nft_can_check() or shutil.which("ip") is None: + self.skipTest("loading a ruleset needs nft with CAP_NET_ADMIN and ip") + for index, path in enumerate(_fragments()): + rel = Path(path).relative_to(PROJECT_ROOT).as_posix() + for mode in MODES: + with self.subTest(fragment=rel, mode=mode): + with ( + tempfile.NamedTemporaryFile("w", suffix=".conf") as handle, + _network_namespace(f"nftprobe{os.getpid()}x{index}") as netns, + ): + handle.write(_render(path, mode)) + handle.flush() + listings = [] + for attempt in ("first", "second"): + loaded = _run([*netns, "nft", "-f", handle.name]) + self.assertEqual( + loaded.returncode, + 0, + f"{rel} failed to load on the {attempt} pass " + f"in {mode}:\n{loaded.stderr}", + ) + listed = _run( + [*netns, "nft", "list", "table", "inet", TABLE] + ) + self.assertEqual( + listed.returncode, + 0, + f"{rel} loaded but left no table on the {attempt} " + f"pass in {mode}:\n{listed.stderr}", + ) + listings.append(listed.stdout) + self.assertIn(f"table inet {TABLE}", listings[0]) + self.assertEqual( + listings[0], + listings[1], + f"{rel} grew on the second pass in {mode}, so it merges " + "into the table it found instead of replacing it", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/roles/svc-net-tor/test_egress_guard_script.py b/tests/unit/python/roles/svc-net-tor/test_egress_guard_script.py deleted file mode 100644 index 2b0fcd7b35..0000000000 --- a/tests/unit/python/roles/svc-net-tor/test_egress_guard_script.py +++ /dev/null @@ -1,110 +0,0 @@ -"""The rendered Tor egress guard rebuilds its chain instead of adding to it. - -The first version of this guard appended: it created the chain when absent and -appended one ACCEPT per admitted range. Narrowing the admitted set then did -nothing, because the retired range had no removal step and the new one landed -below the DROP, where no packet reaches it. The chain could be loosened and -never tightened, which is the wrong direction for a rule that is the only -boundary two policy-less ports have. - -What keeps that from coming back is an order: flush, then admit, then drop, then -hook. These tests pin it against the rendered script, because the ordering is -invisible in the template's individual lines. -""" - -from __future__ import annotations - -import shlex -import unittest -from pathlib import Path - -import jinja2 - -from . import PROJECT_ROOT - -TEMPLATE = ( - PROJECT_ROOT / "roles" / "svc-net-tor" / "templates" / "tor-egress-guard.sh.j2" -) - -CHAIN = "INFINITO_TOR_EGRESS" -CLIENTS = ["127.0.0.0/8", "10.208.0.0/12"] -HOOKS = ["INPUT", "DOCKER-USER"] -PORTS = [ - {"port": "9040", "protocol": "tcp"}, - {"port": "9053", "protocol": "udp"}, -] - - -def _render() -> list[str]: - environment = jinja2.Environment( - keep_trailing_newline=True, - autoescape=False, # noqa: S701 shell, not markup - ) - environment.filters["quote"] = shlex.quote - rendered = environment.from_string( - Path(TEMPLATE).read_text(encoding="utf-8") - ).render( - TOR_EGRESS_GUARD_CHAIN=CHAIN, - TOR_EGRESS_CLIENT_CIDRS=CLIENTS, - TOR_EGRESS_GUARD_HOOK_CHAINS=HOOKS, - TOR_EGRESS_GUARDED_PORTS=PORTS, - ) - return [line.strip() for line in rendered.splitlines() if line.strip()] - - -def _index_of(lines: list[str], needle: str) -> int: - for position, line in enumerate(lines): - if needle in line: - return position - raise AssertionError(f"the rendered guard never {needle!r}:\n" + "\n".join(lines)) - - -class TestEgressGuardScript(unittest.TestCase): - def test_the_chain_is_flushed_before_it_is_filled(self) -> None: - lines = _render() - flush = _index_of(lines, '-F "${CHAIN}"') - first_admit = _index_of(lines, f"-s {CLIENTS[0]}") - self.assertLess( - flush, - first_admit, - "the chain must be flushed before the admitted ranges are added, or a " - "narrowed client list leaves the old range in place", - ) - - def test_every_admitted_range_precedes_the_drop(self) -> None: - lines = _render() - drop = _index_of(lines, "-j DROP") - for cidr in CLIENTS: - with self.subTest(cidr=cidr): - self.assertLess( - _index_of(lines, f"-s {cidr}"), - drop, - f"{cidr} is admitted below the DROP, where it never matches", - ) - - def test_the_hooks_come_after_the_chain_is_complete(self) -> None: - """A hook installed first would route packets into a half-built chain.""" - lines = _render() - drop = _index_of(lines, "-j DROP") - for chain in HOOKS: - for entry in PORTS: - with self.subTest(chain=chain, port=entry["port"]): - hook = _index_of(lines, f"-I {chain} -p {entry['protocol']}") - self.assertGreater(hook, drop) - - def test_every_admitted_range_and_port_is_rendered(self) -> None: - lines = _render() - body = "\n".join(lines) - for cidr in CLIENTS: - self.assertIn(cidr, body) - for chain in HOOKS: - for entry in PORTS: - self.assertIn( - f"--dport {entry['port']}", - body, - f"{chain} never guards {entry['protocol']}/{entry['port']}", - ) - - -if __name__ == "__main__": - unittest.main() From 0ddcf033cde26cbafaaabf16c88ee00a193c796c Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 20:31:20 +0200 Subject: [PATCH 045/197] fix(svc-net-wireguard-firewalled): stop appending a duplicate rule per deploy The role ran one shell line, iptables -A FORWARD -i wg0-client -j ACCEPT followed by iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE. An append checks nothing, so every deploy added the same two rules again and both chains grew for the life of the host. The iptables module probes with -C before it inserts, so the rules converge instead of accumulating, and the two interface names move out of the shell string into variables. The mechanism stays iptables deliberately. An nftables fragment in its own table cannot do this job: an accept there terminates its own base chain and nothing more, so docker's DROP policy on the filter FORWARD chain still executes and the forwarded packet dies. Measured in a namespace with a real forwarding rig: the appended rule delivers 3 of 3 packets under that policy, the fragment delivers 0 of 3, with the fragment's own counter showing the accept matched all three. The inet family would also have started masquerading IPv6, which iptables -t nat never touched. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-net-wireguard-firewalled/README.md | 2 +- .../tasks/00_core.yml | 18 ++++++++++++++++-- .../svc-net-wireguard-firewalled/vars/main.yml | 3 +++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/roles/svc-net-wireguard-firewalled/README.md b/roles/svc-net-wireguard-firewalled/README.md index 90d2eed027..8b42479c3a 100644 --- a/roles/svc-net-wireguard-firewalled/README.md +++ b/roles/svc-net-wireguard-firewalled/README.md @@ -8,7 +8,7 @@ This role adapts iptables rules to enable proper connectivity for a WireGuard cl Optimized for environments with network address translation (NAT), this role: -- Executes shell commands to modify iptables rules. +- Installs its iptables rules through Ansible's `iptables` module, so a redeploy converges instead of appending a duplicate. - Allows traffic from the WireGuard client interface (e.g. `wg0-client`) and sets up NAT masquerading on the external interface (e.g. `eth0`). - Works as an extension to the native WireGuard client role. diff --git a/roles/svc-net-wireguard-firewalled/tasks/00_core.yml b/roles/svc-net-wireguard-firewalled/tasks/00_core.yml index 0c08a0b925..7360c4dc21 100644 --- a/roles/svc-net-wireguard-firewalled/tasks/00_core.yml +++ b/roles/svc-net-wireguard-firewalled/tasks/00_core.yml @@ -1,5 +1,19 @@ --- -- name: ๐Ÿ”ง adapt iptable rules - shell: iptables -A FORWARD -i wg0-client -j ACCEPT && iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE +- name: "๐Ÿงฑ Accept forwarded traffic from the wireguard client" + become: true + ansible.builtin.iptables: + chain: FORWARD + in_interface: "{{ WIREGUARD_CLIENT_INTERFACE }}" + jump: ACCEPT + state: present + +- name: "๐Ÿงฑ Masquerade the client behind the uplink address" + become: true + ansible.builtin.iptables: + table: nat + chain: POSTROUTING + out_interface: "{{ WIREGUARD_UPLINK_INTERFACE }}" + jump: MASQUERADE + state: present - include_tasks: utils/once/flag.yml diff --git a/roles/svc-net-wireguard-firewalled/vars/main.yml b/roles/svc-net-wireguard-firewalled/vars/main.yml index 79d2e17b23..2cd2850160 100644 --- a/roles/svc-net-wireguard-firewalled/vars/main.yml +++ b/roles/svc-net-wireguard-firewalled/vars/main.yml @@ -1,2 +1,5 @@ --- application_id: svc-net-wireguard-firewalled + +WIREGUARD_CLIENT_INTERFACE: wg0-client +WIREGUARD_UPLINK_INTERFACE: eth0 From f9f010b444019c90d1301d67b7b67c868ad7f2b0 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 20:32:35 +0200 Subject: [PATCH 046/197] docs(svc-net-firewall): drop the use case a fragment cannot serve The role advertised holding a forwarding or masquerade rule for a tunnel. A fragment cannot do that: it lives in its own table, and an accept there terminates only its own base chain, so a DROP policy another component set still executes. svc-net-wireguard-firewalled was measured losing every forwarded packet that way. The use case is replaced by one the design does serve, and the rule that decides it is written down where the next author will look rather than left in a commit body. The fragment test no longer seeds the two wireguard interface names; the fragment they described is gone. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-net-firewall/README.md | 4 +++- tests/unit/python/roles/svc-net-firewall/test_fragments.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/svc-net-firewall/README.md b/roles/svc-net-firewall/README.md index a6ef462a55..9556984acb 100644 --- a/roles/svc-net-firewall/README.md +++ b/roles/svc-net-firewall/README.md @@ -36,7 +36,9 @@ Solid `1:1` edges are fixed relationships; dashed `0..1` edges are conditional ( ## Use Cases - Bounding a service port that carries no access policy of its own, where the packet filter is the only boundary available. -- Holding a forwarding or masquerade rule that a tunnel needs, without losing it at the next reboot. +- Redirecting or translating traffic a role owns, scoped to an address range nothing else claims. + +A fragment can only ever be stricter than the host already is. In nftables an `accept` terminates its own base chain and nothing more: the packet still traverses every other chain at that hook, and a `drop` in any of them wins. A rule that has to overrule a policy someone else set, such as the `DROP` docker installs on forwarded packets, does not belong in a fragment at all; it belongs in the chain that carries that policy. ## Quick Setup diff --git a/tests/unit/python/roles/svc-net-firewall/test_fragments.py b/tests/unit/python/roles/svc-net-firewall/test_fragments.py index fc1ce828ee..5156c0f141 100644 --- a/tests/unit/python/roles/svc-net-firewall/test_fragments.py +++ b/tests/unit/python/roles/svc-net-firewall/test_fragments.py @@ -46,8 +46,6 @@ COMMON = { {"port": "9053", "protocol": "tcp"}, {"port": "9053", "protocol": "udp"}, ], - "WIREGUARD_CLIENT_INTERFACE": "wg0-client", - "WIREGUARD_UPLINK_INTERFACE": "eth0", "TOR_EGRESS_VIRTUAL_NET_IPV4": "10.192.0.0/10", "TOR_TRANS_PORT": "9040", "TOR_CONTAINER_DNS_HOST": "172.17.0.1", From 18aaf57006a879665b29c32901f78d269d648e32 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 20:34:33 +0200 Subject: [PATCH 047/197] test(lint): let sys-service own every systemd unit a role installs sys-service resolves the template, writes it into the unit directory, reloads the daemon, enables and starts the unit, and carries the timer, async and flush behaviour. A role that renders its own service.j2 with a bare template task gets the file and none of the rest, and every one of those omissions is silent: a unit that exists but was never enabled looks installed until the host reboots. The lint reports any src pointing at a service.j2 inside a role's tasks. desk-ssh was the one finding left after sys-svc-firewall moved over, and it migrates here: its ssh-agent unit now comes from sys-service in the user scope, with the type, ordering, environment and Install section supplied as template fields. RuntimeMaxSec is pinned empty on purpose, because the default applies it to anything that is not oneshot and would kill a long-running agent after a day. The unversioned unit is retired first. sys-service names units by version, so the old file would otherwise stay behind enabled and a second agent would bind the same socket. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/testing/suppression.md | 1 + roles/desk-ssh/tasks/00_core.yml | 51 ++++++++---- roles/desk-ssh/templates/ssh-agent.service.j2 | 11 --- roles/desk-ssh/vars/main.yml | 2 + .../test_systemd_units_via_sys_service.py | 80 +++++++++++++++++++ 5 files changed, 120 insertions(+), 25 deletions(-) delete mode 100644 roles/desk-ssh/templates/ssh-agent.service.j2 create mode 100644 tests/lint/ansible/modules/test_systemd_units_via_sys_service.py diff --git a/docs/contributing/actions/testing/suppression.md b/docs/contributing/actions/testing/suppression.md index 40bbe4747f..902d77dd07 100644 --- a/docs/contributing/actions/testing/suppression.md +++ b/docs/contributing/actions/testing/suppression.md @@ -108,6 +108,7 @@ The placement rule is per check. The catalog "Position" column uses these labels | `csp-unsafe` | same line | [test_csp_unsafe_requires_justification.py](../../../../tests/lint/ansible/roles/meta/test_csp_unsafe_requires_justification.py) | Documents an `unsafe-*: true` flag in `roles/*/meta/csp.yml`. MUST be paired with a `Reason:` on the same line naming what breaks without the relaxation. `unsafe-inline` re-admits the inline execution an XSS needs and `unsafe-eval` re-admits string-to-code, so the flag has to be verified against the running app, not copied from another role. | | `min-storage-headroom` | same line OR comment block above key | [test_min_storage_headroom.py](../../../../tests/external/update/docker/test_min_storage_headroom.py) | Allows a `roles/*/meta/services.yml` `min_storage` to exceed the measured compressed size of its own `image:`/`version:` by more than 1 GB. MUST be paired with a `Reason:` on the same comment line naming what occupies the space. Reserved for services whose data volume genuinely dominates the footprint; the default path is to lower the value, because an over-declared budget makes a host look too small and prunes a cache the deploy did not need to lose. | | `galaxy-git-parity` | same or above | [test_galaxy_entries_have_git_source.py](../../../../tests/lint/repository/dependencies/test_galaxy_entries_have_git_source.py) | Exempts a `requirements/requirements.galaxy.yml` collection from the requirement to also carry a `type: git` entry in `requirements/requirements.git.yml`. Reserved for collections that publish no usable git remote; without the git entry a `galaxy.ansible.com` outage fails `make install`, because the fallback in `scripts/install/ansible.sh` cannot resolve the collection either. | +| `systemd-via-sys-service` | same or above | [test_systemd_units_via_sys_service.py](../../../../tests/lint/ansible/modules/test_systemd_units_via_sys_service.py) | Allows a role task to render a `*.service.j2` itself instead of handing the unit to `sys-service`. Default path: name the template `templates/systemctl.service.j2` and include `sys-service` with `system_service_id`, so the unit also gets the daemon reload, the enable, and the timer, async and flush behaviour. Reserved for units `sys-service` cannot carry, such as a user-scope unit under `$HOME`. | ## Tool-Native `noqa` Codes ๐Ÿงน diff --git a/roles/desk-ssh/tasks/00_core.yml b/roles/desk-ssh/tasks/00_core.yml index 8996de8785..a6c37710c1 100644 --- a/roles/desk-ssh/tasks/00_core.yml +++ b/roles/desk-ssh/tasks/00_core.yml @@ -21,26 +21,49 @@ - name: โœ… Ensure systemd user directory exists file: - path: "$HOME/.config/systemd/user" + path: "{{ DIR_SYSTEM_SERVICE_USER }}" state: directory mode: "0700" become: false -- name: ๐Ÿš€ Deploy ssh-agent systemd unit file - template: # nocheck: stack-host-template (per-host/user workstation systemd unit) - src: ssh-agent.service.j2 - dest: "$HOME/.config/systemd/user/ssh-agent.service" - mode: "0644" +- name: ๐Ÿ” Look for the unversioned ssh-agent unit + stat: + path: "{{ desk_ssh_legacy_unit }}" + register: desk_ssh_legacy_unit_state become: false -- name: ๐ŸŸข Enable and start ssh-agent service - systemd: - name: ssh-agent.service - scope: user - enabled: true - state: started - daemon_reload: true - become: false +- name: ๐Ÿงน Retire the unversioned ssh-agent unit + block: + - name: ๐Ÿ›‘ Stop and disable the unversioned ssh-agent unit + systemd: + name: "{{ desk_ssh_legacy_unit | basename }}" + scope: user + state: stopped + enabled: false + become: false + + - name: ๐Ÿ—‘๏ธ Delete the unversioned ssh-agent unit + file: + path: "{{ desk_ssh_legacy_unit }}" + state: absent + become: false + when: desk_ssh_legacy_unit_state.stat.exists + +- name: ๐Ÿš€ Deploy ssh-agent systemd unit file + include_role: + name: sys-service + vars: + system_service_scope: user + system_service_copy_files: false + system_service_state: started + system_service_tpl_on_failure: "" + system_service_tpl_type: simple + system_service_tpl_runtime: "" + system_service_tpl_before: default.target + system_service_tpl_wanted_by: default.target + system_service_tpl_environment: + - "SSH_AUTH_SOCK=%t/ssh-agent.socket" + system_service_tpl_exec_start: "/usr/bin/ssh-agent -D -a $SSH_AUTH_SOCK" - name: โœ… Ensure ~/.profile exists with common environment lineinfile: diff --git a/roles/desk-ssh/templates/ssh-agent.service.j2 b/roles/desk-ssh/templates/ssh-agent.service.j2 deleted file mode 100644 index 3125724f18..0000000000 --- a/roles/desk-ssh/templates/ssh-agent.service.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=User SSH Agent -Before=default.target - -[Service] -Type=simple -Environment=SSH_AUTH_SOCK=%t/ssh-agent.socket -ExecStart=/usr/bin/ssh-agent -D -a $SSH_AUTH_SOCK - -[Install] -WantedBy=default.target \ No newline at end of file diff --git a/roles/desk-ssh/vars/main.yml b/roles/desk-ssh/vars/main.yml index 7d866e2329..d8bb7879e0 100644 --- a/roles/desk-ssh/vars/main.yml +++ b/roles/desk-ssh/vars/main.yml @@ -1,3 +1,5 @@ --- application_id: desk-ssh +system_service_id: "{{ application_id }}" desk_ssh_repository: "{{ lookup('config', application_id, 'services.ssh.repository') }}" +desk_ssh_legacy_unit: "{{ [ DIR_SYSTEM_SERVICE_USER, 'ssh-agent.service' ] | path_join }}" diff --git a/tests/lint/ansible/modules/test_systemd_units_via_sys_service.py b/tests/lint/ansible/modules/test_systemd_units_via_sys_service.py new file mode 100644 index 0000000000..4edb292bfa --- /dev/null +++ b/tests/lint/ansible/modules/test_systemd_units_via_sys_service.py @@ -0,0 +1,80 @@ +"""Lint: a systemd unit is installed through sys-service, not by hand. + +``sys-service`` owns what it takes to make a unit real: resolving the template, +writing it into the system unit directory, reloading the daemon, enabling and +starting it, the timer variant, and the async and flush behaviour the deploy +relies on. A role that renders its own ``*.service.j2`` with a bare ``template:`` +gets the file and none of the rest, and every one of those omissions is silent โ€” +a unit that exists but was never enabled looks installed until the host reboots. + +It is also how the same decisions end up answered differently per role. That is +not hypothetical: the role this lint was written alongside rendered its own unit +and its own reload handler before anyone noticed sys-service already did both. + +Suppression (see ``docs/contributing/actions/testing/suppression.md``): + +* ``# nocheck: systemd-via-sys-service`` on the ``src:`` line or anywhere in the + task body, for a unit sys-service genuinely cannot carry. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +from utils.annotations.suppress import is_suppressed_at +from utils.cache.files import iter_project_files_with_content + +from . import PROJECT_ROOT + +_RULE = "systemd-via-sys-service" +_OWNER = "sys-service" +_UNIT_SRC = re.compile(r"^\s*src:\s*[\"']?(?P[^\"'\s]+\.service\.j2)[\"']?") + + +def _is_task_file(rel_path: str) -> bool: + return ( + rel_path.startswith("roles/") + and "/tasks/" in rel_path + and rel_path.endswith((".yml", ".yaml")) + ) + + +def _role_of(rel_path: str) -> str: + return Path(rel_path).parts[1] + + +class TestSystemdUnitsViaSysService(unittest.TestCase): + def test_no_role_installs_its_own_unit(self) -> None: + findings: list[str] = [] + for path_str, content in iter_project_files_with_content( + extensions=(".yml", ".yaml"), + exclude_tests=True, + ): + rel = Path(path_str).relative_to(PROJECT_ROOT).as_posix() + if not _is_task_file(rel) or _role_of(rel) == _OWNER: + continue + lines = content.splitlines() + for index, line in enumerate(lines): + match = _UNIT_SRC.match(line) + if match is None: + continue + if is_suppressed_at(lines, index + 1, _RULE, mode="same-or-above"): + continue + findings.append(f"- {rel}:{index + 1}: {match.group('src')}") + + if findings: + self.fail( + "These roles render a systemd unit themselves instead of handing " + "it to sys-service, which also reloads the daemon, enables the " + "unit, and carries the timer, async and flush behaviour:\n" + + "\n".join(sorted(set(findings))) + + "\n\nFix: name the template templates/systemctl.service.j2 and " + "include the sys-service role with system_service_id, or mark the " + f"task `# nocheck: {_RULE}` when sys-service cannot carry it." + ) + + +if __name__ == "__main__": + unittest.main() From 508bc5adac8dbe8f4997fcbbe421969f30f26581 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 20:36:04 +0200 Subject: [PATCH 048/197] test(lint): keep shell scripts out of jinja templates A templated script is opaque to every tool that reads shell. shellcheck parses Jinja as syntax errors, the repository's own .sh guards skip the file because it is not a .sh, and nothing can execute it to test it: what runs on the host never exists in the repository, so a defect in the rendered half is found by deploying it. The lint reports any *.sh.j2 under roles and points at the static slot sys-service already resolves, files/shell/script.sh, with the values passed as ExecStart arguments or Environment entries the way this repository's python scripts already take them. None of the seven templates present meets the bar the marker is for, which is a script whose shape is generated rather than its values. Each is grandfathered with a marker naming what it interpolates, because converting a deployed script to argv changes what runs on the host. The guard exists to stop the next one. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/testing/suppression.md | 1 + .../templates/script.sh.j2 | 1 + .../templates/script.sh.j2 | 1 + .../templates/script.sh.j2 | 1 + .../sys-ctl-alm-email/templates/script.sh.j2 | 1 + .../templates/script.sh.j2 | 1 + .../sys-ctl-cln-docker/templates/script.sh.j2 | 1 + roles/user-backup/templates/ssh-wrapper.sh.j2 | 1 + .../shell/test_no_shell_templates.py | 64 +++++++++++++++++++ 9 files changed, 72 insertions(+) create mode 100644 tests/lint/filesystem/shell/test_no_shell_templates.py diff --git a/docs/contributing/actions/testing/suppression.md b/docs/contributing/actions/testing/suppression.md index 902d77dd07..fa4bac1300 100644 --- a/docs/contributing/actions/testing/suppression.md +++ b/docs/contributing/actions/testing/suppression.md @@ -108,6 +108,7 @@ The placement rule is per check. The catalog "Position" column uses these labels | `csp-unsafe` | same line | [test_csp_unsafe_requires_justification.py](../../../../tests/lint/ansible/roles/meta/test_csp_unsafe_requires_justification.py) | Documents an `unsafe-*: true` flag in `roles/*/meta/csp.yml`. MUST be paired with a `Reason:` on the same line naming what breaks without the relaxation. `unsafe-inline` re-admits the inline execution an XSS needs and `unsafe-eval` re-admits string-to-code, so the flag has to be verified against the running app, not copied from another role. | | `min-storage-headroom` | same line OR comment block above key | [test_min_storage_headroom.py](../../../../tests/external/update/docker/test_min_storage_headroom.py) | Allows a `roles/*/meta/services.yml` `min_storage` to exceed the measured compressed size of its own `image:`/`version:` by more than 1 GB. MUST be paired with a `Reason:` on the same comment line naming what occupies the space. Reserved for services whose data volume genuinely dominates the footprint; the default path is to lower the value, because an over-declared budget makes a host look too small and prunes a cache the deploy did not need to lose. | | `galaxy-git-parity` | same or above | [test_galaxy_entries_have_git_source.py](../../../../tests/lint/repository/dependencies/test_galaxy_entries_have_git_source.py) | Exempts a `requirements/requirements.galaxy.yml` collection from the requirement to also carry a `type: git` entry in `requirements/requirements.git.yml`. Reserved for collections that publish no usable git remote; without the git entry a `galaxy.ansible.com` outage fails `make install`, because the fallback in `scripts/install/ansible.sh` cannot resolve the collection either. | +| `shell-template` | head (first 30 lines) | [test_no_shell_templates.py](../../../../tests/lint/filesystem/shell/test_no_shell_templates.py) | Allows a `*.sh.j2` to exist. Default path: ship the script as a plain `.sh` under `files/shell/` and pass its values as `ExecStart` arguments or `Environment=` entries, because shellcheck, the repository's `.sh` guards and every test skip a Jinja template, so nothing reads what actually runs on the host. Reserved for a script whose shape is generated, not just its values. Place the marker BELOW the shebang, never above it, and prefer the Jinja comment form so it does not reach the rendered script. | | `systemd-via-sys-service` | same or above | [test_systemd_units_via_sys_service.py](../../../../tests/lint/ansible/modules/test_systemd_units_via_sys_service.py) | Allows a role task to render a `*.service.j2` itself instead of handing the unit to `sys-service`. Default path: name the template `templates/systemctl.service.j2` and include `sys-service` with `system_service_id`, so the unit also gets the daemon reload, the enable, and the timer, async and flush behaviour. Reserved for units `sys-service` cannot carry, such as a user-scope unit under `$HOME`. | ## Tool-Native `noqa` Codes ๐Ÿงน diff --git a/roles/svc-bkp-remote-2-local/templates/script.sh.j2 b/roles/svc-bkp-remote-2-local/templates/script.sh.j2 index 0000b7c862..4cdffc1dc8 100644 --- a/roles/svc-bkp-remote-2-local/templates/script.sh.j2 +++ b/roles/svc-bkp-remote-2-local/templates/script.sh.j2 @@ -1,4 +1,5 @@ #!/bin/bash +{# nocheck: shell-template grandfathered, values only: the provider list and two paths #} # Pulls the remote backups from multiple hosts hosts="{{ DOCKER_BACKUP_REMOTE_2_LOCAL_BACKUP_PROVIDERS | join(' ') }}" diff --git a/roles/svc-net-wireguard-plain/templates/script.sh.j2 b/roles/svc-net-wireguard-plain/templates/script.sh.j2 index 2a0bf75c2d..5412c5143e 100644 --- a/roles/svc-net-wireguard-plain/templates/script.sh.j2 +++ b/roles/svc-net-wireguard-plain/templates/script.sh.j2 @@ -1,4 +1,5 @@ #!/bin/bash +{# nocheck: shell-template grandfathered: one line per internet interface, replaceable by argv #} {% for internet_interface in internet_interfaces %} ip li set mtu 1400 dev {{internet_interface}} {% endfor %} diff --git a/roles/sys-ctl-alm-compose/templates/script.sh.j2 b/roles/sys-ctl-alm-compose/templates/script.sh.j2 index d9007ff9d5..ff85bcf637 100644 --- a/roles/sys-ctl-alm-compose/templates/script.sh.j2 +++ b/roles/sys-ctl-alm-compose/templates/script.sh.j2 @@ -1,4 +1,5 @@ #!/bin/bash +{# nocheck: shell-template grandfathered: one start block per subservice, each naming a unit only unit_name can resolve #} err=0 set -u start_timeout="{{ SYSTEMCTL_ALARM_COMPOSER_START_TIMEOUT_SECONDS }}" diff --git a/roles/sys-ctl-alm-email/templates/script.sh.j2 b/roles/sys-ctl-alm-email/templates/script.sh.j2 index b8b89bf89d..c9ec0bdf61 100644 --- a/roles/sys-ctl-alm-email/templates/script.sh.j2 +++ b/roles/sys-ctl-alm-email/templates/script.sh.j2 @@ -1,4 +1,5 @@ #!/bin/bash +{# nocheck: shell-template grandfathered, values only: the mail envelope, charset and timeout #} set -u # Reverse systemd escaping for human-readable mail subject/body friendly="$(systemd-escape --unescape "$1")" diff --git a/roles/sys-ctl-alm-telegram/templates/script.sh.j2 b/roles/sys-ctl-alm-telegram/templates/script.sh.j2 index 32a1fb0f94..726a842e4f 100644 --- a/roles/sys-ctl-alm-telegram/templates/script.sh.j2 +++ b/roles/sys-ctl-alm-telegram/templates/script.sh.j2 @@ -1,4 +1,5 @@ #!/bin/bash +{# nocheck: shell-template grandfathered, values only: bot token, chat id and the curl filter's timeout #} set -u # Make the instance text human readable again (reverse systemd escaping) diff --git a/roles/sys-ctl-cln-docker/templates/script.sh.j2 b/roles/sys-ctl-cln-docker/templates/script.sh.j2 index 4a7bd76017..5d66f0f6d7 100644 --- a/roles/sys-ctl-cln-docker/templates/script.sh.j2 +++ b/roles/sys-ctl-cln-docker/templates/script.sh.j2 @@ -1,4 +1,5 @@ #!/bin/sh +{# nocheck: shell-template grandfathered, values only: the versioned unit name it starts #} # Cleans up anonymous Docker volumes and performs a full Docker system prune. set -e diff --git a/roles/user-backup/templates/ssh-wrapper.sh.j2 b/roles/user-backup/templates/ssh-wrapper.sh.j2 index 8ef003fc56..86e03d9763 100644 --- a/roles/user-backup/templates/ssh-wrapper.sh.j2 +++ b/roles/user-backup/templates/ssh-wrapper.sh.j2 @@ -1,4 +1,5 @@ #!/bin/sh +{# nocheck: shell-template grandfathered, values only: the backup directory and the repository list #} # log command if [ -n "$SSH_ORIGINAL_COMMAND" ] diff --git a/tests/lint/filesystem/shell/test_no_shell_templates.py b/tests/lint/filesystem/shell/test_no_shell_templates.py new file mode 100644 index 0000000000..512c1656c5 --- /dev/null +++ b/tests/lint/filesystem/shell/test_no_shell_templates.py @@ -0,0 +1,64 @@ +"""Lint: a shell script ships as ``.sh``, not as ``.sh.j2``. + +A templated script is opaque to every tool that reads shell. ``shellcheck`` +parses Jinja as syntax errors, the repo's own ``.sh`` guards (``|| true``, +``sh -c pipefail``, inline PHP) skip the file because it is not a ``.sh``, and +nothing can execute it to test it: what runs on the host never exists in the +repository. A defect in the rendered half is found by deploying it. + +The variability belongs in the invocation, not in the text. ``sys-service`` +resolves ``files/shell/script.sh`` alongside ``templates/script.sh.j2``, and a +static script takes its values the way this repo's Python scripts already do: +as arguments on ``ExecStart``, or through ``Environment=``. + +Suppression (see ``docs/contributing/actions/testing/suppression.md``): + +* ``nocheck: shell-template`` anywhere in the first 30 lines, for a script whose + shape itself is generated rather than its values. Put the marker BELOW the + shebang, never above it, and prefer the Jinja comment form ``{# ... #}`` so it + does not land in the rendered script. State what is generated. + +None of the templates present when this guard landed meets that bar: each +interpolates values into a fixed script, and each is grandfathered with a marker +saying so, because converting a deployed script to argv changes what runs on the +host. The guard exists to stop the next one. +""" + +from __future__ import annotations + +import unittest +from pathlib import Path + +from utils.annotations.suppress import is_suppressed_in_head +from utils.cache.files import iter_project_files, read_text + +from . import PROJECT_ROOT + +_RULE = "shell-template" +_SUFFIX = ".sh.j2" + + +class TestNoShellTemplates(unittest.TestCase): + def test_no_shell_script_is_a_template(self) -> None: + findings: list[str] = [] + for path in iter_project_files(extensions=(_SUFFIX,), exclude_tests=True): + lines = read_text(path).splitlines() + if is_suppressed_in_head(lines, _RULE): + continue + findings.append(f"- {Path(path).relative_to(PROJECT_ROOT).as_posix()}") + + if findings: + self.fail( + "These shell scripts are Jinja templates, so shellcheck, the " + "repository's .sh guards and every test read something other " + "than what runs on the host:\n" + + "\n".join(sorted(findings)) + + "\n\nFix: move the script to files/shell/ as a plain .sh and " + "pass its values as ExecStart arguments or Environment= entries, " + f"or mark it `nocheck: {_RULE}` in the file head when the script's " + "shape, not just its values, is generated." + ) + + +if __name__ == "__main__": + unittest.main() From c89ca42c463aeda7473737426844acd6fb076c62 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 20:53:14 +0200 Subject: [PATCH 049/197] fix(svc-net-tor): bound the two ports that answered everyone An audit of what the host actually filters found a single input chain in the whole ruleset, covering three tor ports. ss showed two more binds with nothing in front of them: the SOCKS port on 0.0.0.0, and dnsmasq on the bridge address. Each gets the layer it belongs in. SOCKS gets a packet filter rule against the same CIDRs SocksPolicy already accepts, so no client loses access while the filter closes if tor's own config ever stops enforcing. Reusing the narrower guard set was deliberately avoided: it excludes the LAN blocks SocksPolicy admits, so it would have changed the policy rather than hardened it. dnsmasq gets local-service, which answers only directly connected networks. A packet filter rule there would have sat on the container DNS path, which daemon.json points at that same bridge address, in exchange for closing an exposure that needs an extra route onto the bridge. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-net-tor/templates/dnsmasq-tor-onion.conf.j2 | 1 + roles/svc-net-tor/templates/nftables.conf.j2 | 7 +++++++ roles/svc-net-tor/vars/main.yml | 1 + tests/unit/python/roles/svc-net-firewall/test_fragments.py | 7 +++++++ 4 files changed, 16 insertions(+) diff --git a/roles/svc-net-tor/templates/dnsmasq-tor-onion.conf.j2 b/roles/svc-net-tor/templates/dnsmasq-tor-onion.conf.j2 index b073b59cee..2644a9fa60 100644 --- a/roles/svc-net-tor/templates/dnsmasq-tor-onion.conf.j2 +++ b/roles/svc-net-tor/templates/dnsmasq-tor-onion.conf.j2 @@ -4,6 +4,7 @@ listen-address=127.0.0.1 listen-address={{ TOR_CONTAINER_DNS_HOST }} {% endif %} bind-interfaces +local-service no-resolv {% endif %} no-hosts diff --git a/roles/svc-net-tor/templates/nftables.conf.j2 b/roles/svc-net-tor/templates/nftables.conf.j2 index 69155b8a43..cf1979bf3a 100644 --- a/roles/svc-net-tor/templates/nftables.conf.j2 +++ b/roles/svc-net-tor/templates/nftables.conf.j2 @@ -8,10 +8,17 @@ table inet {{ FIREWALL_TABLE }} { elements = { {{ TOR_EGRESS_CLIENT_CIDRS | join(', ') }} } } + set socks_admitted { + type ipv4_addr + flags interval + elements = { {{ NETWORK_INTERNAL_CIDRS | join(', ') }} } + } + chain guard { {% for entry in TOR_EGRESS_GUARDED_PORTS %} ip saddr != @admitted {{ entry.protocol }} dport {{ entry.port }} drop {% endfor %} + ip saddr != @socks_admitted tcp dport {{ TOR_SOCKS_PORT }} drop } chain input { diff --git a/roles/svc-net-tor/vars/main.yml b/roles/svc-net-tor/vars/main.yml index 24ae0ebfa5..a02e9f2d67 100644 --- a/roles/svc-net-tor/vars/main.yml +++ b/roles/svc-net-tor/vars/main.yml @@ -11,6 +11,7 @@ TOR_USE_ENTRY_GUARDS: "{{ lookup('config', application_id, 'services.tor.use_ent TOR_FLAVOR: "{{ lookup('config', application_id, 'services.tor.flavor') }}" TOR_TRANS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.trans') }}" TOR_DNS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.dns') }}" +TOR_SOCKS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.socks') }}" TOR_DNSMASQ_CONF: "/etc/dnsmasq.d/tor-onion.conf" TOR_EGRESS_GUARDED_PORTS: diff --git a/tests/unit/python/roles/svc-net-firewall/test_fragments.py b/tests/unit/python/roles/svc-net-firewall/test_fragments.py index 5156c0f141..bc7828ab8f 100644 --- a/tests/unit/python/roles/svc-net-firewall/test_fragments.py +++ b/tests/unit/python/roles/svc-net-firewall/test_fragments.py @@ -48,7 +48,14 @@ COMMON = { ], "TOR_EGRESS_VIRTUAL_NET_IPV4": "10.192.0.0/10", "TOR_TRANS_PORT": "9040", + "TOR_SOCKS_PORT": "9050", "TOR_CONTAINER_DNS_HOST": "172.17.0.1", + "NETWORK_INTERNAL_CIDRS": [ + "127.0.0.0/8", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + ], } MODES = ("compose", "swarm") From 06cc485995018086daba700e49569d7e5b18e1f4 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 21:04:42 +0200 Subject: [PATCH 050/197] feat(svc-net-tor): reject a forged source on the guarded ports The guard trusted an address. rp_filter ships at all=0 and default=2 on this host and the repository sets it nowhere, so loose mode accepts a packet whose source is routable via any interface at all. A datagram claiming a docker pool address and arriving on the uplink passed both the kernel check and the admitted set, which for udp needs one forged packet and no handshake. A strict reverse-path check on the guarded ports closes it. Measured in a namespace rig: a forged source routable via another interface reaches the input hook, because loose rp_filter lets it through, and the new rule counts it; a source routable nowhere never gets that far, the kernel drops it first. A loopback datagram to the same port does not match, so the host's own .onion resolution through 127.0.0.1 to the DNSPort is untouched. Scoped to the four guarded ports rather than the whole chain. Strict reverse-path filtering on everything traversing input and forward breaks asymmetric routing, and this role runs on hosts that terminate wireguard tunnels. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-net-tor/templates/nftables.conf.j2 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/svc-net-tor/templates/nftables.conf.j2 b/roles/svc-net-tor/templates/nftables.conf.j2 index cf1979bf3a..974ae97a63 100644 --- a/roles/svc-net-tor/templates/nftables.conf.j2 +++ b/roles/svc-net-tor/templates/nftables.conf.j2 @@ -14,11 +14,19 @@ table inet {{ FIREWALL_TABLE }} { elements = { {{ NETWORK_INTERNAL_CIDRS | join(', ') }} } } +{% set _tcp_guarded = ((TOR_EGRESS_GUARDED_PORTS | selectattr('protocol', 'equalto', 'tcp') | map(attribute='port') | list) + [TOR_SOCKS_PORT]) | unique | list %} +{% set _udp_guarded = TOR_EGRESS_GUARDED_PORTS | selectattr('protocol', 'equalto', 'udp') | map(attribute='port') | unique | list %} chain guard { {% for entry in TOR_EGRESS_GUARDED_PORTS %} ip saddr != @admitted {{ entry.protocol }} dport {{ entry.port }} drop {% endfor %} ip saddr != @socks_admitted tcp dport {{ TOR_SOCKS_PORT }} drop +{% if _tcp_guarded | length > 0 %} + fib saddr . iif oif missing tcp dport { {{ _tcp_guarded | join(', ') }} } drop +{% endif %} +{% if _udp_guarded | length > 0 %} + fib saddr . iif oif missing udp dport { {{ _udp_guarded | join(', ') }} } drop +{% endif %} } chain input { From 612830920708cdb54eee5f55e10d8a4399199a91 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 21:13:18 +0200 Subject: [PATCH 051/197] test(svc-net-tor): pin the address space the egress guard has to admit The guard drops every source outside the list tor_egress_clients returns, and containers reach the TransPort and the DNSPort through exactly those rules. Nothing held that relationship: narrowing a pool, renaming a key or dropping an entry from the lookup would have taken the resolver and the transparent proxy away from every container at once, in production. The test runs the real lookup over the repository's real network declarations rather than a fixture, so it fails where the mistake is made. Its templar resolves the cross references those declarations carry, because without it the lookup would compare unrendered template text and the check would pass on nothing. It also pins the decision that the admitted list is deliberately narrower than SocksPolicy: 10.0.0.0/8 stays out, because a transparent proxy and a resolver with no access control of their own do not belong to the whole LAN. Left unwritten, that reads like an oversight and gets repaired on the next pass. Co-Authored-By: Claude Opus 5 (1M context) --- .../roles/svc-net-tor/test_egress_clients.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 tests/unit/python/roles/svc-net-tor/test_egress_clients.py diff --git a/tests/unit/python/roles/svc-net-tor/test_egress_clients.py b/tests/unit/python/roles/svc-net-tor/test_egress_clients.py new file mode 100644 index 0000000000..5fbc87f0b6 --- /dev/null +++ b/tests/unit/python/roles/svc-net-tor/test_egress_clients.py @@ -0,0 +1,99 @@ +"""The egress guard admits every address a container can come from. + +The guard drops any source outside ``tor_egress_clients``, and containers reach +the TransPort and the DNSPort through exactly those rules. So the list is not a +convenience: if it ever stops covering docker's own address pools, every +container loses the resolver and the transparent proxy at once, and it fails in +production rather than here. + +This runs the real lookup over the repository's real network declarations. A +narrowed pool, a renamed key or a dropped entry in the lookup all surface as a +failure of the property the rules depend on. +""" + +from __future__ import annotations + +import importlib.util +import sys +import unittest +from unittest import mock + +import jinja2 + +from utils.cache.yaml import load_yaml_any + +from . import PROJECT_ROOT + +NETWORKS = PROJECT_ROOT / "group_vars/all/08_networks.yml" +LOOKUP = "roles/svc-net-tor/lookup_plugins/tor_egress_clients.py" + + +def _load_lookup(): + path = PROJECT_ROOT / LOOKUP + spec = importlib.util.spec_from_file_location("tor_egress_clients_lookup", path) + module = importlib.util.module_from_spec(spec) + assert spec and spec.loader + sys.modules["tor_egress_clients_lookup"] = module + spec.loader.exec_module(module) + return module + + +class _Templar: + """Resolves the group_vars cross-references the lookup renders.""" + + def __init__(self, variables: dict): + self.available_variables = variables + + def template(self, value): + return ( + jinja2.Environment(autoescape=False) # noqa: S701 - renders CIDRs, not markup + .from_string(str(value)) + .render(**self.available_variables) + ) + + +class TestEgressClients(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.module = _load_lookup() + cls.variables = load_yaml_any(str(NETWORKS)) + + def _admitted(self) -> list[str]: + lookup = self.module.LookupModule() + lookup._templar = _Templar(self.variables) + lookup._loader = mock.MagicMock() + with mock.patch.object(self.module, "lookup_loader") as loader: + loader.get.return_value = mock.MagicMock(run=mock.MagicMock(return_value=[[]])) + return lookup.run([], variables=self.variables)[0] + + def test_every_docker_address_pool_is_admitted(self) -> None: + admitted = self._admitted() + pools = self.variables["NETWORK_DOCKER_ADDRESS_POOLS"] + self.assertTrue(pools, "no docker address pool is declared") + for pool in pools: + with self.subTest(pool=pool["base"]): + self.assertIn( + pool["base"], + admitted, + "containers draw their address from this pool and reach the " + "guarded ports through the guard, so dropping it from the " + "admitted list takes the resolver away from every container", + ) + + def test_loopback_is_admitted(self) -> None: + """The host resolves .onion through its own loopback.""" + self.assertIn(self.variables["NETWORK_LOOPBACK_CIDR"], self._admitted()) + + def test_the_lan_is_not_admitted_wholesale(self) -> None: + """The guarded ports carry no access policy, so the LAN is not a client. + + Admitting 10.0.0.0/8 would hand a transparent proxy and a resolver to + every host the operator's network happens to contain. + """ + self.assertNotIn( + self.variables["NETWORK_PRIVATE_CIDRS"]["class_a"], self._admitted() + ) + + +if __name__ == "__main__": + unittest.main() From 154d0c0cd2d4a98b080e9f2402ec4202787c94b0 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 21:14:10 +0200 Subject: [PATCH 052/197] test(lint): refuse an accept verdict in a firewall fragment A fragment declares its own nftables table. An accept there terminates that table's base chain and nothing else: the packet still traverses every other base chain at the same hook, and a drop in any of them wins. Docker sets the filter FORWARD policy to drop by default, so a fragment that accepts is not permitting anything, it is declining to decide. Measured rather than argued. Converting a role's appended iptables FORWARD accept into a fragment rule at priority -10 delivered 0 of 3 forwarded packets under that policy where the rule it replaced delivered 3 of 3, and the fragment's own counter showed the accept had matched all three. Both versions deployed green, which is why this belongs in a lint and not in a paragraph. The message names both ways out: express the intent as a drop of what must not pass, or put the rule in the chain that carries the policy it has to overrule. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/testing/suppression.md | 1 + .../jinja/test_firewall_fragment_verdicts.py | 79 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 tests/lint/ansible/jinja/test_firewall_fragment_verdicts.py diff --git a/docs/contributing/actions/testing/suppression.md b/docs/contributing/actions/testing/suppression.md index fa4bac1300..71ef5b652c 100644 --- a/docs/contributing/actions/testing/suppression.md +++ b/docs/contributing/actions/testing/suppression.md @@ -108,6 +108,7 @@ The placement rule is per check. The catalog "Position" column uses these labels | `csp-unsafe` | same line | [test_csp_unsafe_requires_justification.py](../../../../tests/lint/ansible/roles/meta/test_csp_unsafe_requires_justification.py) | Documents an `unsafe-*: true` flag in `roles/*/meta/csp.yml`. MUST be paired with a `Reason:` on the same line naming what breaks without the relaxation. `unsafe-inline` re-admits the inline execution an XSS needs and `unsafe-eval` re-admits string-to-code, so the flag has to be verified against the running app, not copied from another role. | | `min-storage-headroom` | same line OR comment block above key | [test_min_storage_headroom.py](../../../../tests/external/update/docker/test_min_storage_headroom.py) | Allows a `roles/*/meta/services.yml` `min_storage` to exceed the measured compressed size of its own `image:`/`version:` by more than 1 GB. MUST be paired with a `Reason:` on the same comment line naming what occupies the space. Reserved for services whose data volume genuinely dominates the footprint; the default path is to lower the value, because an over-declared budget makes a host look too small and prunes a cache the deploy did not need to lose. | | `galaxy-git-parity` | same or above | [test_galaxy_entries_have_git_source.py](../../../../tests/lint/repository/dependencies/test_galaxy_entries_have_git_source.py) | Exempts a `requirements/requirements.galaxy.yml` collection from the requirement to also carry a `type: git` entry in `requirements/requirements.git.yml`. Reserved for collections that publish no usable git remote; without the git entry a `galaxy.ansible.com` outage fails `make install`, because the fallback in `scripts/install/ansible.sh` cannot resolve the collection either. | +| `firewall-fragment-accept` | same or above | [test_firewall_fragment_verdicts.py](../../../../tests/lint/ansible/jinja/test_firewall_fragment_verdicts.py) | Allows a rule in a `roles/*/templates/nftables.conf.j2` fragment to end in an `accept` verdict. Default path: express the intent as a `drop` of what must not pass, or put the rule in the chain carrying the policy it has to overrule, because an `accept` terminates only its own base chain and every other chain at that hook still runs, docker's `drop` policy on forwarded packets included. Reserved for an accept whose only job is to skip the fragment's own later rules. | | `shell-template` | head (first 30 lines) | [test_no_shell_templates.py](../../../../tests/lint/filesystem/shell/test_no_shell_templates.py) | Allows a `*.sh.j2` to exist. Default path: ship the script as a plain `.sh` under `files/shell/` and pass its values as `ExecStart` arguments or `Environment=` entries, because shellcheck, the repository's `.sh` guards and every test skip a Jinja template, so nothing reads what actually runs on the host. Reserved for a script whose shape is generated, not just its values. Place the marker BELOW the shebang, never above it, and prefer the Jinja comment form so it does not reach the rendered script. | | `systemd-via-sys-service` | same or above | [test_systemd_units_via_sys_service.py](../../../../tests/lint/ansible/modules/test_systemd_units_via_sys_service.py) | Allows a role task to render a `*.service.j2` itself instead of handing the unit to `sys-service`. Default path: name the template `templates/systemctl.service.j2` and include `sys-service` with `system_service_id`, so the unit also gets the daemon reload, the enable, and the timer, async and flush behaviour. Reserved for units `sys-service` cannot carry, such as a user-scope unit under `$HOME`. | diff --git a/tests/lint/ansible/jinja/test_firewall_fragment_verdicts.py b/tests/lint/ansible/jinja/test_firewall_fragment_verdicts.py new file mode 100644 index 0000000000..7c0fefc9a8 --- /dev/null +++ b/tests/lint/ansible/jinja/test_firewall_fragment_verdicts.py @@ -0,0 +1,79 @@ +"""Lint: a firewall fragment may tighten, never overrule. + +A fragment declares its own nftables table. In nftables an ``accept`` verdict +terminates that table's base chain and nothing else: the packet still traverses +every other base chain registered at the same hook, and a ``drop`` in any of +them wins. Docker sets the filter FORWARD policy to ``drop`` by default, so a +fragment that accepts is not permitting anything, it is only declining to +decide. + +Measured, not assumed: converting a role's ``iptables -A FORWARD -i wg0 -j +ACCEPT`` into a fragment rule at priority -10 dropped every forwarded packet +under that policy, 0 of 3 against 3 of 3 for the rule it replaced, with the +fragment's own counter showing the accept had matched all three. The deploy was +green either way. + +A rule that has to overrule a policy another component set belongs in the chain +that carries that policy, not in a fragment. + +Suppression (see ``docs/contributing/actions/testing/suppression.md``): + +* ``nocheck: firewall-fragment-accept`` on the rule's line or the one above it, + for an accept whose only job is to skip the fragment's own later rules. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +from utils.annotations.suppress import is_suppressed_at +from utils.cache.files import iter_project_files, read_text + +from . import PROJECT_ROOT + +_RULE = "firewall-fragment-accept" +_FRAGMENT = "templates/nftables.conf.j2" +_POLICY = re.compile(r"\bpolicy\s+accept\b") +_VERDICT = re.compile(r"\baccept\s*$") + + +def _fragments() -> list[str]: + suffix = "/" + _FRAGMENT + return sorted( + path + for path in iter_project_files(extensions=(".j2",), exclude_tests=True) + if path.endswith(suffix) + ) + + +class TestFirewallFragmentVerdicts(unittest.TestCase): + def test_no_fragment_rule_accepts(self) -> None: + findings: list[str] = [] + for path in _fragments(): + rel = Path(path).relative_to(PROJECT_ROOT).as_posix() + lines = read_text(path).splitlines() + for index, line in enumerate(lines): + body = line.split("#")[0].rstrip().rstrip(";") + if _POLICY.search(body) or not _VERDICT.search(body): + continue + if is_suppressed_at(lines, index + 1, _RULE, mode="same-or-above"): + continue + findings.append(f"- {rel}:{index + 1}: {line.strip()}") + + if findings: + self.fail( + "These fragment rules end in an accept verdict, which terminates " + "only this table's base chain. Another chain at the same hook can " + "still drop the packet, so the rule permits nothing:\n" + + "\n".join(findings) + + "\n\nFix: express the intent as a drop of what must not pass, or " + "put the rule in the chain that carries the policy it has to " + f"overrule. Mark it `nocheck: {_RULE}` when the accept only skips " + "the fragment's own later rules." + ) + + +if __name__ == "__main__": + unittest.main() From f19d5e34774ecf21cb3740181dfa7e87fdd7dd78 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 21:23:17 +0200 Subject: [PATCH 053/197] test(lint): refuse a loop_var that shadows a declared variable Ansible warns that the name is already in use and carries on. Carrying on is the problem: inside the loop the original value is gone, so every lookup, template and nested include reading that name sees the loop item instead, and the task still reports success. The damage lands in a log nobody greps. The lint collects every key declared in group_vars and in each role's vars or defaults, then reports any loop_var matching one. A denylist of known-bad names would have found application_id and missed domain, which is the same class in four more places. All seven existing sites are marked, each with the reason read at the source rather than inferred. The three application_id loops pass the role id as the item, and the include_vars that follows loads that role's own application_id with the same value, so both meanings coincide by construction. The four domain loops hand the vhost to sys-stk-front-proxy, whose tasks read front_proxy_domain from exactly that name, so the shadowing is the interface. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/testing/suppression.md | 1 + roles/test-e2e-cli/tasks/main.yml | 2 +- .../tasks/01_setup/orchestrate.yml | 4 +- roles/web-app-joomla/tasks/main.yml | 2 +- roles/web-app-mastodon/tasks/00_core.yml | 2 +- roles/web-app-peertube/tasks/00_core.yml | 2 +- roles/web-app-wordpress/tasks/00_core.yml | 2 +- .../ansible/tasks/test_loop_var_shadowing.py | 103 ++++++++++++++++++ 8 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 tests/lint/ansible/tasks/test_loop_var_shadowing.py diff --git a/docs/contributing/actions/testing/suppression.md b/docs/contributing/actions/testing/suppression.md index 71ef5b652c..69130b7a4a 100644 --- a/docs/contributing/actions/testing/suppression.md +++ b/docs/contributing/actions/testing/suppression.md @@ -108,6 +108,7 @@ The placement rule is per check. The catalog "Position" column uses these labels | `csp-unsafe` | same line | [test_csp_unsafe_requires_justification.py](../../../../tests/lint/ansible/roles/meta/test_csp_unsafe_requires_justification.py) | Documents an `unsafe-*: true` flag in `roles/*/meta/csp.yml`. MUST be paired with a `Reason:` on the same line naming what breaks without the relaxation. `unsafe-inline` re-admits the inline execution an XSS needs and `unsafe-eval` re-admits string-to-code, so the flag has to be verified against the running app, not copied from another role. | | `min-storage-headroom` | same line OR comment block above key | [test_min_storage_headroom.py](../../../../tests/external/update/docker/test_min_storage_headroom.py) | Allows a `roles/*/meta/services.yml` `min_storage` to exceed the measured compressed size of its own `image:`/`version:` by more than 1 GB. MUST be paired with a `Reason:` on the same comment line naming what occupies the space. Reserved for services whose data volume genuinely dominates the footprint; the default path is to lower the value, because an over-declared budget makes a host look too small and prunes a cache the deploy did not need to lose. | | `galaxy-git-parity` | same or above | [test_galaxy_entries_have_git_source.py](../../../../tests/lint/repository/dependencies/test_galaxy_entries_have_git_source.py) | Exempts a `requirements/requirements.galaxy.yml` collection from the requirement to also carry a `type: git` entry in `requirements/requirements.git.yml`. Reserved for collections that publish no usable git remote; without the git entry a `galaxy.ansible.com` outage fails `make install`, because the fallback in `scripts/install/ansible.sh` cannot resolve the collection either. | +| `loop-var-shadow` | same or above | [test_loop_var_shadowing.py](../../../../tests/lint/ansible/tasks/test_loop_var_shadowing.py) | Allows a `loop_var` to take the name of a variable declared in `group_vars/**` or in a role's `vars`/`defaults`. Inside the loop that name carries the item, so every lookup, template and nested include reading it sees the item instead, and ansible only warns. Reserved for a loop whose body reads the name as the item on purpose, typically because the item IS that value: state which meaning the body relies on. | | `firewall-fragment-accept` | same or above | [test_firewall_fragment_verdicts.py](../../../../tests/lint/ansible/jinja/test_firewall_fragment_verdicts.py) | Allows a rule in a `roles/*/templates/nftables.conf.j2` fragment to end in an `accept` verdict. Default path: express the intent as a `drop` of what must not pass, or put the rule in the chain carrying the policy it has to overrule, because an `accept` terminates only its own base chain and every other chain at that hook still runs, docker's `drop` policy on forwarded packets included. Reserved for an accept whose only job is to skip the fragment's own later rules. | | `shell-template` | head (first 30 lines) | [test_no_shell_templates.py](../../../../tests/lint/filesystem/shell/test_no_shell_templates.py) | Allows a `*.sh.j2` to exist. Default path: ship the script as a plain `.sh` under `files/shell/` and pass its values as `ExecStart` arguments or `Environment=` entries, because shellcheck, the repository's `.sh` guards and every test skip a Jinja template, so nothing reads what actually runs on the host. Reserved for a script whose shape is generated, not just its values. Place the marker BELOW the shebang, never above it, and prefer the Jinja comment form so it does not reach the rendered script. | | `systemd-via-sys-service` | same or above | [test_systemd_units_via_sys_service.py](../../../../tests/lint/ansible/modules/test_systemd_units_via_sys_service.py) | Allows a role task to render a `*.service.j2` itself instead of handing the unit to `sys-service`. Default path: name the template `templates/systemctl.service.j2` and include `sys-service` with `system_service_id`, so the unit also gets the daemon reload, the enable, and the timer, async and flush behaviour. Reserved for units `sys-service` cannot carry, such as a user-scope unit under `$HOME`. | diff --git a/roles/test-e2e-cli/tasks/main.yml b/roles/test-e2e-cli/tasks/main.yml index 78e2190aea..cfb3092bf3 100644 --- a/roles/test-e2e-cli/tasks/main.yml +++ b/roles/test-e2e-cli/tasks/main.yml @@ -44,7 +44,7 @@ ansible.builtin.include_tasks: run_one.yml loop: "{{ _cli_apps }}" loop_control: - loop_var: application_id + loop_var: application_id # nocheck: loop-var-shadow the item IS the role id, so run_one reads the same value the tested role declares label: "{{ application_id }}" - name: โœ… Fail if there were CLI test failures diff --git a/roles/test-e2e-playwright/tasks/01_setup/orchestrate.yml b/roles/test-e2e-playwright/tasks/01_setup/orchestrate.yml index 0a0db3cd4b..9580f19f34 100644 --- a/roles/test-e2e-playwright/tasks/01_setup/orchestrate.yml +++ b/roles/test-e2e-playwright/tasks/01_setup/orchestrate.yml @@ -18,7 +18,7 @@ ansible.builtin.include_tasks: 02_run_one.yml loop: "{{ TEST_E2E_PLAYWRIGHT_APPS }}" loop_control: - loop_var: application_id + loop_var: application_id # nocheck: loop-var-shadow the item IS the role id, so 02_run_one reads the same value the tested role's own vars declare label: "{{ application_id }}" - name: ๐Ÿ” Stash first-pass failures for one flake retry @@ -31,7 +31,7 @@ ansible.builtin.include_tasks: 02_run_one.yml loop: "{{ _playwright_retry_roles | default([]) }}" loop_control: - loop_var: application_id + loop_var: application_id # nocheck: loop-var-shadow same loop body, same identity between item and role id label: "{{ application_id }} (retry)" - name: ๐Ÿ›‘ Fail if there were Playwright failures diff --git a/roles/web-app-joomla/tasks/main.yml b/roles/web-app-joomla/tasks/main.yml index d26681fb8d..0c2adc2ad8 100644 --- a/roles/web-app-joomla/tasks/main.yml +++ b/roles/web-app-joomla/tasks/main.yml @@ -4,7 +4,7 @@ name: sys-stk-front-proxy loop: "{{ JOOMLA_DOMAINS }}" loop_control: - loop_var: domain + loop_var: domain # nocheck: loop-var-shadow sys-stk-front-proxy takes the vhost through this name, see its tasks/main.yml front_proxy_domain vars: http_port: "{{ lookup('config', application_id, 'services.joomla.ports.local.http') }}" webserver_client_max_body_size: "{{ JOOMLA_POST_MAX_SIZE }}" diff --git a/roles/web-app-mastodon/tasks/00_core.yml b/roles/web-app-mastodon/tasks/00_core.yml index 53b877980b..4e60f56b9b 100644 --- a/roles/web-app-mastodon/tasks/00_core.yml +++ b/roles/web-app-mastodon/tasks/00_core.yml @@ -6,7 +6,7 @@ name: sys-stk-front-proxy loop: "{{ lookup('domains', 'web-app-mastodon') }}" loop_control: - loop_var: domain + loop_var: domain # nocheck: loop-var-shadow sys-stk-front-proxy takes the vhost through this name, see its tasks/main.yml front_proxy_domain vars: http_port: "{{ lookup('config', application_id, 'services.mastodon.ports.local.http') }}" webserver_websocket_location: "/api/v1/streaming" diff --git a/roles/web-app-peertube/tasks/00_core.yml b/roles/web-app-peertube/tasks/00_core.yml index 596c18e492..e4aa750ef9 100644 --- a/roles/web-app-peertube/tasks/00_core.yml +++ b/roles/web-app-peertube/tasks/00_core.yml @@ -6,7 +6,7 @@ name: sys-stk-front-proxy loop: "{{ lookup('domains', application_id) }}" loop_control: - loop_var: domain + loop_var: domain # nocheck: loop-var-shadow sys-stk-front-proxy takes the vhost through this name, see its tasks/main.yml front_proxy_domain vars: http_port: "{{ lookup('config', application_id, 'services.peertube.ports.local.http') }}" websocket_port: "{{ lookup('config', application_id, 'services.peertube.ports.local.http') }}" diff --git a/roles/web-app-wordpress/tasks/00_core.yml b/roles/web-app-wordpress/tasks/00_core.yml index 4aa2ed8fa5..2e3806e1a2 100644 --- a/roles/web-app-wordpress/tasks/00_core.yml +++ b/roles/web-app-wordpress/tasks/00_core.yml @@ -10,7 +10,7 @@ name: sys-stk-front-proxy loop: "{{ WORDPRESS_DOMAINS }}" loop_control: - loop_var: domain + loop_var: domain # nocheck: loop-var-shadow sys-stk-front-proxy takes the vhost through this name, see its tasks/main.yml front_proxy_domain vars: webserver_extra_configuration: "client_max_body_size {{ WORDPRESS_MAX_UPLOAD_SIZE }};" http_port: "{{ lookup('config', application_id, 'services.wordpress.ports.local.http') }}" diff --git a/tests/lint/ansible/tasks/test_loop_var_shadowing.py b/tests/lint/ansible/tasks/test_loop_var_shadowing.py new file mode 100644 index 0000000000..414434dad4 --- /dev/null +++ b/tests/lint/ansible/tasks/test_loop_var_shadowing.py @@ -0,0 +1,103 @@ +"""Lint: a ``loop_var`` may not take the name of a declared variable. + +Ansible warns and carries on: + + [WARNING]: The variable 'application_id' is already in use. + You should set the `loop_var` value in the `loop_control` option for the + task to something else to avoid variable collisions and unexpected + behavior. + +Carrying on is the problem. Inside the loop the original value is gone, so +every lookup, template and nested include that reads the name sees the loop +item instead. The task still succeeds, which is why this shows up as a warning +in a log nobody greps rather than as a failure. + +``application_id`` is the sharp case here: almost every role declares it and +``include_role`` carries it into whatever the loop body reaches, so a loop over +applications that names its item ``application_id`` silently repoints every +config lookup in the nested role. + +Declared means: a key in ``group_vars/**`` or in any role's ``vars/main.yml`` +or ``defaults/main.yml``. Cross-role includes are routine in this repository, +so a name another role declares is a name that can be in scope here. + +Suppression (see ``docs/contributing/actions/testing/suppression.md``): + +* ``nocheck: loop-var-shadow`` on the ``loop_var`` line or the one above it, + when the shadowing is deliberate and the loop body provably reads no other + meaning of the name. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +from utils.annotations.suppress import is_suppressed_at +from utils.cache.files import iter_project_files, read_text +from utils.cache.yaml import load_yaml_any + +from . import PROJECT_ROOT + +_RULE = "loop-var-shadow" +_LOOP_VAR = re.compile(r"^\s*loop_var:\s*[\"']?(?P[A-Za-z_][A-Za-z0-9_]*)") + + +def _declared_names() -> set[str]: + """Every variable name the project declares in a file Ansible loads.""" + names: set[str] = set() + for path in iter_project_files(extensions=(".yml", ".yaml"), exclude_tests=True): + rel = Path(path).relative_to(PROJECT_ROOT).as_posix() + is_group_var = rel.startswith("group_vars/") + is_role_var = re.fullmatch(r"roles/[^/]+/(vars|defaults)/main\.ya?ml", rel) + if not (is_group_var or is_role_var): + continue + loaded = load_yaml_any(path) + if isinstance(loaded, dict): + names.update(key for key in loaded if isinstance(key, str)) + return names + + +def _task_files() -> list[str]: + return sorted( + path + for path in iter_project_files(extensions=(".yml", ".yaml"), exclude_tests=True) + if "/tasks/" in Path(path).relative_to(PROJECT_ROOT).as_posix() + ) + + +class TestLoopVarShadowing(unittest.TestCase): + def test_no_loop_var_shadows_a_declared_variable(self) -> None: + declared = _declared_names() + self.assertTrue( + declared, "no variable declarations were found to compare against" + ) + + findings: list[str] = [] + for path in _task_files(): + rel = Path(path).relative_to(PROJECT_ROOT).as_posix() + lines = read_text(path).splitlines() + for index, line in enumerate(lines): + match = _LOOP_VAR.match(line) + if match is None or match.group("name") not in declared: + continue + if is_suppressed_at(lines, index + 1, _RULE, mode="same-or-above"): + continue + findings.append(f"- {rel}:{index + 1}: loop_var: {match.group('name')}") + + if findings: + self.fail( + "These loops name their item after a variable the project " + "declares, so inside the loop that variable carries the item " + "instead of its own value and every nested lookup reads the " + "wrong thing:\n" + + "\n".join(findings) + + "\n\nFix: give the loop item a name of its own, for example " + f"`loop_var: _item`, or mark it `nocheck: {_RULE}` when the " + "loop body provably reads no other meaning of the name." + ) + + +if __name__ == "__main__": + unittest.main() From a5d3ff21cfe868e113151c2b76219a031a92ba29 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 21:24:35 +0200 Subject: [PATCH 054/197] style: apply the formatter to the egress client test The line that mocks the role_subnets dependency exceeded the line length; the project formatter broke it across three. No behaviour changes. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/python/roles/svc-net-tor/test_egress_clients.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/python/roles/svc-net-tor/test_egress_clients.py b/tests/unit/python/roles/svc-net-tor/test_egress_clients.py index 5fbc87f0b6..52a558a9a9 100644 --- a/tests/unit/python/roles/svc-net-tor/test_egress_clients.py +++ b/tests/unit/python/roles/svc-net-tor/test_egress_clients.py @@ -63,7 +63,9 @@ class TestEgressClients(unittest.TestCase): lookup._templar = _Templar(self.variables) lookup._loader = mock.MagicMock() with mock.patch.object(self.module, "lookup_loader") as loader: - loader.get.return_value = mock.MagicMock(run=mock.MagicMock(return_value=[[]])) + loader.get.return_value = mock.MagicMock( + run=mock.MagicMock(return_value=[[]]) + ) return lookup.run([], variables=self.variables)[0] def test_every_docker_address_pool_is_admitted(self) -> None: From aa6bbe3b20e45ecb054d0333ec2b80a7f4df0c62 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 15 Sep 2026 21:38:45 +0200 Subject: [PATCH 055/197] feat(lint): render the software identity from its declared variables SOFTWARE_NAME, SOFTWARE_DOMAIN, SOFTWARE_URL and ORGANIZATION are declared once in group_vars/all/00_general.yml and derive from each other. Eleven templates wrote the value instead, so a rename would reach the four declarations and every consumer that reads them, and silently miss the literal. The lint reads the name from the declaration rather than carrying it, and searches case-insensitively: the domain is the name lowered, the URL is https:// plus the domain and the organization is the name again, so one literal finds the value of all four. A hit is reported with the variable it should have been, derived from the scheme in front of it and its case. The escape is nocheck: software-identity, same line or the one above. Every migrated template renders the same bytes as before. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/testing/suppression.md | 1 + roles/svc-net-firewall/templates/main.nft.j2 | 2 +- roles/svc-net-tor/templates/torrc.j2 | 2 +- .../templates/exports.j2 | 2 +- .../templates/javascript/oidc.js.j2 | 2 +- roles/web-app-jenkins/templates/casc.yaml.j2 | 2 +- .../templates/zzz_auth_ldap.xml.j2 | 8 +- .../templates/html/nav/bottom.html.j2 | 4 +- .../templates/javascript.js.j2 | 4 +- .../templates/playwright.env.j2 | 2 +- .../web-app-mediawiki/templates/style.css.j2 | 2 +- .../templates/playwright.env.j2 | 2 +- .../jinja/test_software_identity_variables.py | 91 +++++++++++++++++++ 13 files changed, 108 insertions(+), 16 deletions(-) create mode 100644 tests/lint/ansible/jinja/test_software_identity_variables.py diff --git a/docs/contributing/actions/testing/suppression.md b/docs/contributing/actions/testing/suppression.md index 69130b7a4a..48661f5814 100644 --- a/docs/contributing/actions/testing/suppression.md +++ b/docs/contributing/actions/testing/suppression.md @@ -112,6 +112,7 @@ The placement rule is per check. The catalog "Position" column uses these labels | `firewall-fragment-accept` | same or above | [test_firewall_fragment_verdicts.py](../../../../tests/lint/ansible/jinja/test_firewall_fragment_verdicts.py) | Allows a rule in a `roles/*/templates/nftables.conf.j2` fragment to end in an `accept` verdict. Default path: express the intent as a `drop` of what must not pass, or put the rule in the chain carrying the policy it has to overrule, because an `accept` terminates only its own base chain and every other chain at that hook still runs, docker's `drop` policy on forwarded packets included. Reserved for an accept whose only job is to skip the fragment's own later rules. | | `shell-template` | head (first 30 lines) | [test_no_shell_templates.py](../../../../tests/lint/filesystem/shell/test_no_shell_templates.py) | Allows a `*.sh.j2` to exist. Default path: ship the script as a plain `.sh` under `files/shell/` and pass its values as `ExecStart` arguments or `Environment=` entries, because shellcheck, the repository's `.sh` guards and every test skip a Jinja template, so nothing reads what actually runs on the host. Reserved for a script whose shape is generated, not just its values. Place the marker BELOW the shebang, never above it, and prefer the Jinja comment form so it does not reach the rendered script. | | `systemd-via-sys-service` | same or above | [test_systemd_units_via_sys_service.py](../../../../tests/lint/ansible/modules/test_systemd_units_via_sys_service.py) | Allows a role task to render a `*.service.j2` itself instead of handing the unit to `sys-service`. Default path: name the template `templates/systemctl.service.j2` and include `sys-service` with `system_service_id`, so the unit also gets the daemon reload, the enable, and the timer, async and flush behaviour. Reserved for units `sys-service` cannot carry, such as a user-scope unit under `$HOME`. | +| `software-identity` | same or above | [test_software_identity_variables.py](../../../../tests/lint/ansible/jinja/test_software_identity_variables.py) | Allows a `.j2` template to write the software's name, domain or URL as a literal. Default path: render `SOFTWARE_NAME`, `SOFTWARE_DOMAIN`, `SOFTWARE_URL` or `ORGANIZATION`, which `group_vars/all/00_general.yml` declares once and derives from each other, so a rename reaches the template too. Reserved for a literal that must survive a rename, such as an identifier a third party issued under the old name. | ## Tool-Native `noqa` Codes ๐Ÿงน diff --git a/roles/svc-net-firewall/templates/main.nft.j2 b/roles/svc-net-firewall/templates/main.nft.j2 index 466e80c23c..ef967514c8 100644 --- a/roles/svc-net-firewall/templates/main.nft.j2 +++ b/roles/svc-net-firewall/templates/main.nft.j2 @@ -1,3 +1,3 @@ #!/usr/sbin/nft -f -# Managed by Infinito.Nexus (roles/svc-net-firewall). Do not edit by hand. +# Managed by {{ SOFTWARE_NAME }} (roles/svc-net-firewall). Do not edit by hand. include "{{ FIREWALL_FRAGMENT_DIR }}/*.conf" diff --git a/roles/svc-net-tor/templates/torrc.j2 b/roles/svc-net-tor/templates/torrc.j2 index bcaf9564ae..20e1dddbbb 100644 --- a/roles/svc-net-tor/templates/torrc.j2 +++ b/roles/svc-net-tor/templates/torrc.j2 @@ -1,4 +1,4 @@ -# Managed by Infinito.Nexus (roles/svc-net-tor) โ€” do not edit by hand. +# Managed by {{ SOFTWARE_NAME }} (roles/svc-net-tor) โ€” do not edit by hand. RunAsDaemon 0 Log notice stdout DataDirectory {{ lookup('volume', 'svc-net-tor', 'tor_data').target }} diff --git a/roles/svc-storage-nfs-server/templates/exports.j2 b/roles/svc-storage-nfs-server/templates/exports.j2 index 471a61869d..830dfceee0 100644 --- a/roles/svc-storage-nfs-server/templates/exports.j2 +++ b/roles/svc-storage-nfs-server/templates/exports.j2 @@ -1,4 +1,4 @@ -# Managed by Infinito.Nexus Ansible role svc-storage-nfs-server. +# Managed by {{ SOFTWARE_NAME }} Ansible role svc-storage-nfs-server. {% for ip in svc_storage_nfs_server_allowed_ips %} {{ STORAGE_NFS_STATE_PATH }} {{ ip }}({{ SVC_STORAGE_NFS_SERVER_STATE_EXPORT_OPTIONS }},fsid=0) {% endfor %} diff --git a/roles/web-app-dashboard/templates/javascript/oidc.js.j2 b/roles/web-app-dashboard/templates/javascript/oidc.js.j2 index 2d641ae521..52b28a94a4 100644 --- a/roles/web-app-dashboard/templates/javascript/oidc.js.j2 +++ b/roles/web-app-dashboard/templates/javascript/oidc.js.j2 @@ -8,7 +8,7 @@ 1) Configuration (filled by Jinja/Ansible) ======================= */ const KC_CONFIG = { - url: "{{ OIDC.URL }}", // e.g. https://auth.infinito.nexus + url: "{{ OIDC.URL }}", // e.g. https://auth.{{ SOFTWARE_DOMAIN }} realm: "{{ OIDC.CLIENT.REALM }}", // e.g. cymais.cloud clientId: "{{ OIDC.CLIENT.ID }}", // e.g. cymais.cloud redirectUri: "{{ DASHBOARD_APP_BASE_URL }}", // canonical app URL for login/logout round-trips diff --git a/roles/web-app-jenkins/templates/casc.yaml.j2 b/roles/web-app-jenkins/templates/casc.yaml.j2 index b40e863375..e66a916a60 100644 --- a/roles/web-app-jenkins/templates/casc.yaml.j2 +++ b/roles/web-app-jenkins/templates/casc.yaml.j2 @@ -5,7 +5,7 @@ # Keycloak group / LDAP group memberships onto Jenkins authorities. jenkins: - systemMessage: "Managed by Infinito.Nexus." + systemMessage: "Managed by {{ SOFTWARE_NAME }}." numExecutors: 2 mode: NORMAL diff --git a/roles/web-app-kix/templates/zzz_auth_ldap.xml.j2 b/roles/web-app-kix/templates/zzz_auth_ldap.xml.j2 index a5cd11055c..209028dcf8 100644 --- a/roles/web-app-kix/templates/zzz_auth_ldap.xml.j2 +++ b/roles/web-app-kix/templates/zzz_auth_ldap.xml.j2 @@ -1,12 +1,12 @@ - Infinito.Nexus LDAP backend (svc-db-openldap) for the web-app-kix-administrator role group. Authenticates KIX agents via Auth::LDAP and provisions the contact via Auth::Sync::LDAP. + {{ SOFTWARE_NAME }} LDAP backend (svc-db-openldap) for the web-app-kix-administrator role group. Authenticates KIX agents via Auth::LDAP and provisions the contact via Auth::Sync::LDAP. Framework [ { - "Name": "Infinito.Nexus svc-db-openldap (Agent, web-app-kix-administrator)", + "Name": "{{ SOFTWARE_NAME }} svc-db-openldap (Agent, web-app-kix-administrator)", "Enabled": 1, "UsageContext": "Agent", "Module": "Kernel::System::Auth::LDAP", @@ -57,12 +57,12 @@ - Second Infinito.Nexus LDAP backend for the web-app-kix-user role group. KIX iterates over Authentication entries; if the administrator backend (###100) refuses (user not in admin group), this one tries the user group. The OAuth2-proxy `allowed_groups` filter still lets only members of either group through, so this backend never sees a user without an Infinito.Nexus role. + Second {{ SOFTWARE_NAME }} LDAP backend for the web-app-kix-user role group. KIX iterates over Authentication entries; if the administrator backend (###100) refuses (user not in admin group), this one tries the user group. The OAuth2-proxy `allowed_groups` filter still lets only members of either group through, so this backend never sees a user without an {{ SOFTWARE_NAME }} role. Framework [ { - "Name": "Infinito.Nexus svc-db-openldap (Agent, web-app-kix-user)", + "Name": "{{ SOFTWARE_NAME }} svc-db-openldap (Agent, web-app-kix-user)", "Enabled": 1, "UsageContext": "Agent", "Module": "Kernel::System::Auth::LDAP", diff --git a/roles/web-app-littlejs/templates/html/nav/bottom.html.j2 b/roles/web-app-littlejs/templates/html/nav/bottom.html.j2 index 709d70be95..94b2eac154 100644 --- a/roles/web-app-littlejs/templates/html/nav/bottom.html.j2 +++ b/roles/web-app-littlejs/templates/html/nav/bottom.html.j2 @@ -7,13 +7,13 @@ LittleJS Repository - + {{ LITTLEJS_HEADLINE }} Code
- + {{ SOFTWARE_NAME }}
diff --git a/roles/web-app-mattermost/templates/javascript.js.j2 b/roles/web-app-mattermost/templates/javascript.js.j2 index b96dc1378e..4dec5e539a 100644 --- a/roles/web-app-mattermost/templates/javascript.js.j2 +++ b/roles/web-app-mattermost/templates/javascript.js.j2 @@ -5,7 +5,7 @@ window.addEventListener("DOMContentLoaded", function () { // Rename if Mattermost ever renders the GitLab button natively document.querySelectorAll("a[href*='/oauth/gitlab/']").forEach(function (el) { if (el.textContent.trim() === "GitLab") { - el.textContent = "SSO with Infinito.Nexus"; + el.textContent = "SSO with {{ SOFTWARE_NAME }}"; } }); @@ -19,7 +19,7 @@ window.addEventListener("DOMContentLoaded", function () { var a = document.createElement("a"); a.href = "/oauth/gitlab/login"; - a.textContent = "SSO with Infinito.Nexus"; + a.textContent = "SSO with {{ SOFTWARE_NAME }}"; a.className = "btn btn-primary login-body-card-form-button-submit large"; a.style.cssText = "display:flex;align-items:center;justify-content:center;width:100%;box-sizing:border-box;margin:20px 0 0 0;"; btn.parentNode.insertBefore(a, btn.nextSibling); diff --git a/roles/web-app-mattermost/templates/playwright.env.j2 b/roles/web-app-mattermost/templates/playwright.env.j2 index 44e5be9c6e..4f807ed924 100644 --- a/roles/web-app-mattermost/templates/playwright.env.j2 +++ b/roles/web-app-mattermost/templates/playwright.env.j2 @@ -36,7 +36,7 @@ MCP_SERVICE_ENABLED={{ lookup('config', application_id, 'mcp.enabled') | string OIDC_ISSUER_URL={{ OIDC.CLIENT.ISSUER_URL | dotenv_quote }} -# nocheck: persona-block-rationale โ€” Mattermost Team Edition has no native OIDC provider, so this role piggybacks Keycloak onto the GitLab OAuth slot (vars/main.yml, MM_GITLABSETTINGS_* in templates/env.j2); the entry point is a[href='/oauth/gitlab/login'] relabelled "SSO with Infinito.Nexus" by templates/javascript.js.j2, which the shared helper's name-based login matcher does not match, and Mattermost v11+ first serves a /landing interstitial that only clears once localStorage.__landingPageSeen__ is seeded before navigation โ€” the shared helper has no init-script hook (see README "Persona contract opt-outs"). +# nocheck: persona-block-rationale โ€” Mattermost Team Edition has no native OIDC provider, so this role piggybacks Keycloak onto the GitLab OAuth slot (vars/main.yml, MM_GITLABSETTINGS_* in templates/env.j2); the entry point is a[href='/oauth/gitlab/login'] relabelled "SSO with {{ SOFTWARE_NAME }}" by templates/javascript.js.j2, which the shared helper's name-based login matcher does not match, and Mattermost v11+ first serves a /landing interstitial that only clears once localStorage.__landingPageSeen__ is seeded before navigation โ€” the shared helper has no init-script hook (see README "Persona contract opt-outs"). PERSONA_ADMINISTRATOR_BLOCKED=true # nocheck: persona-block-rationale โ€” see PERSONA_ADMINISTRATOR_BLOCKED above. diff --git a/roles/web-app-mediawiki/templates/style.css.j2 b/roles/web-app-mediawiki/templates/style.css.j2 index b5b5fcfb1d..68379bc125 100644 --- a/roles/web-app-mediawiki/templates/style.css.j2 +++ b/roles/web-app-mediawiki/templates/style.css.j2 @@ -1,4 +1,4 @@ -/* MediaWiki (Vector 2022 + OOUI) color remap via Infinito.Nexus palette +/* MediaWiki (Vector 2022 + OOUI) color remap via {{ SOFTWARE_NAME }} palette - Keep changes scoped and variable-driven - Prefer variables over hard values - Avoid !important unless an inline style or strong OOUI rule requires it diff --git a/roles/web-app-pixelfed/templates/playwright.env.j2 b/roles/web-app-pixelfed/templates/playwright.env.j2 index 7cf2bd23f0..df0ab60782 100644 --- a/roles/web-app-pixelfed/templates/playwright.env.j2 +++ b/roles/web-app-pixelfed/templates/playwright.env.j2 @@ -7,7 +7,7 @@ ADMIN_USERNAME={{ lookup('users', 'administrator').username | dotenv_quote }} APP_BASE_URL={{ lookup('tls', application_id, 'url.base') }} -BIBER_OIDC_EMAIL_OVERRIDE={{ (lookup('users', 'biber').username ~ '@infinito.nexus') | dotenv_quote }} +BIBER_OIDC_EMAIL_OVERRIDE={{ (lookup('users', 'biber').username ~ '@' ~ SOFTWARE_DOMAIN) | dotenv_quote }} BIBER_PASSWORD={{ lookup('users', 'biber').password | dotenv_quote }} diff --git a/tests/lint/ansible/jinja/test_software_identity_variables.py b/tests/lint/ansible/jinja/test_software_identity_variables.py new file mode 100644 index 0000000000..dde53d1aef --- /dev/null +++ b/tests/lint/ansible/jinja/test_software_identity_variables.py @@ -0,0 +1,91 @@ +"""Lint: a template spells the software's identity through its variables. + +``SOFTWARE_NAME``, ``SOFTWARE_DOMAIN``, ``SOFTWARE_URL`` and ``ORGANIZATION`` +are declared once in ``group_vars/all/00_general.yml`` and derive from each +other. A template that writes the value instead of the variable forks that +single point: a rename reaches the four declarations and every consumer that +reads them, and silently misses the literal. + +The check needs one literal only. ``SOFTWARE_DOMAIN`` is ``SOFTWARE_NAME | +lower``, ``SOFTWARE_URL`` is ``https://`` plus the domain and ``ORGANIZATION`` +is the name again, so a case-insensitive search for the name's value finds the +value of all four. The value is read from the declaration rather than written +here, so renaming the software does not leave this lint hunting the old string. + +Suppression (see ``docs/contributing/actions/testing/suppression.md``): + +* ``nocheck: software-identity`` on the offending line or the one above it, + for a literal that must survive a rename, such as an identifier a third + party already issued under the old name. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +from utils.annotations.suppress import is_suppressed_at +from utils.cache.files import iter_project_files, read_text +from utils.cache.yaml import load_yaml_any + +from . import PROJECT_ROOT + +_RULE = "software-identity" +_SPOT = "group_vars/all/00_general.yml" +_NAME_KEY = "SOFTWARE_NAME" + + +def _software_name() -> str: + """The literal value of ``SOFTWARE_NAME``, read from its declaration.""" + declared = load_yaml_any(str(PROJECT_ROOT / _SPOT)) + if not isinstance(declared, dict) or _NAME_KEY not in declared: + raise AssertionError(f"{_SPOT} declares no {_NAME_KEY}") + name = declared[_NAME_KEY] + if not isinstance(name, str) or "{{" in name: + raise AssertionError( + f"{_SPOT}: {_NAME_KEY} is no longer a plain literal ({name!r}), so " + "this lint can no longer derive what to search for" + ) + return name + + +def _variable_for(line: str, start: int, matched: str) -> str: + if line[:start].endswith(("https://", "http://")): + return "SOFTWARE_URL" + if matched.islower(): + return "SOFTWARE_DOMAIN" + return f"{_NAME_KEY} (or ORGANIZATION)" + + +class TestSoftwareIdentityVariables(unittest.TestCase): + def test_no_template_spells_out_the_software_identity(self) -> None: + pattern = re.compile(re.escape(_software_name()), re.IGNORECASE) + + findings: list[str] = [] + for path in iter_project_files(extensions=(".j2",), exclude_tests=True): + rel = Path(path).relative_to(PROJECT_ROOT).as_posix() + lines = read_text(path).splitlines() + for index, line in enumerate(lines): + match = pattern.search(line) + if match is None: + continue + if is_suppressed_at(lines, index + 1, _RULE, mode="same-or-above"): + continue + variable = _variable_for(line, match.start(), match.group(0)) + findings.append( + f"- {rel}:{index + 1}: {match.group(0)!r} -> {{{{ {variable} }}}}" + ) + + if findings: + self.fail( + "These templates write the software's identity as a literal, so a " + f"rename in {_SPOT} would leave them behind:\n" + + "\n".join(findings) + + f"\n\nFix: render the declared variable instead. Mark it " + f"`nocheck: {_RULE}` when the literal must survive a rename." + ) + + +if __name__ == "__main__": + unittest.main() From abb6f37297117e20892c74b7f21eca68a4205027 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 00:21:58 +0200 Subject: [PATCH 056/197] fix(svc-net-tor): drop the dnsmasq keyword the RedHat family already sets EL10 and Fedora ship an active local-service=host in /etc/dnsmasq.conf. The drop-in repeated the keyword, and dnsmasq refuses a repeated ARG_ONE option: 'illegal repeated keyword at line N of /etc/dnsmasq.d//tor-onion.conf', then 'FAILED to start up'. Eight CI rows died at roles/svc-net-tor/tasks/router.yml:67, all of them on centos or fedora; arch and ubuntu start dnsmasq fine because their stock config leaves the keyword commented. The line never took effect on any distribution. dnsmasq resets OPT_LOCAL_SERVICE as soon as an access-control option is present, and listen-address fills exactly that list; the template emits listen-address=127.0.0.1 unconditionally in the same block. A query from a source outside every directly attached subnet is answered identically with and without the line. Port 53 therefore carries no source restriction. That is unchanged here and belongs in the nftables guard, which currently covers only the trans, DNS and SOCKS ports of tor itself. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-net-tor/templates/dnsmasq-tor-onion.conf.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/svc-net-tor/templates/dnsmasq-tor-onion.conf.j2 b/roles/svc-net-tor/templates/dnsmasq-tor-onion.conf.j2 index 2644a9fa60..b073b59cee 100644 --- a/roles/svc-net-tor/templates/dnsmasq-tor-onion.conf.j2 +++ b/roles/svc-net-tor/templates/dnsmasq-tor-onion.conf.j2 @@ -4,7 +4,6 @@ listen-address=127.0.0.1 listen-address={{ TOR_CONTAINER_DNS_HOST }} {% endif %} bind-interfaces -local-service no-resolv {% endif %} no-hosts From b5807e66ce960cc7236bf34c2cb6b553a07ae8c4 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 01:26:03 +0200 Subject: [PATCH 057/197] fix(svc-net-tor): guard the node's resolver against everything it routes dnsmasq binds the docker bridge address with the clearnet upstreams configured and the node's own .onion records loaded, and no rule covered port 53. Any source that could route a packet to that address got an open resolver plus the split-DNS records. c89ca42c46 claimed local-service closed this; abb6f37297 showed the keyword never took effect. The guard now carries two named sets instead of one plus a special case. 'containers' is the docker pool list, for tor's own transparent and DNS ports. 'internal' is NETWORK_INTERNAL_CIDRS, for the resolver and the SOCKS port. Every guarded entry names its set, which folds the hand-written SOCKS rule and its manual append into the same list the reverse-path sets are built from. The resolver takes the wider set deliberately: 'internal' is the list torrc.j2 already spends on SocksPolicy accept. The narrow set carries only 10.208.0.0/12 and 10.7.7.0/24 out of 10.0.0.0/8, which leaves out swarm's ingress network, every overlay docker allocates without an ipam block, and a node whose own address sits elsewhere in 10/8. A query bound to such an address was measured timing out against the narrow set. Guarding the resolver more strictly than the proxy beside it would refuse a name lookup to a source that is handed a full SOCKS proxy six lines earlier. NETWORK_DNS_PORT lives next to the other resolver constants rather than in meta/services.yml: ports.local describes host-bound container ports, where 53 collides with svc-dns-unbound and falls outside every PORT_BANDS band. Measured on the rendered artifact: the table loads and reloads with a byte-identical listing, every compose source of port 53 resolves inside 'internal' and every source of 9040/9053 inside 'containers', and a mistyped set name, an empty CIDR list or a missing admitted key each fail loudly at validate time. Port 53 newly passes a strict reverse-path check, so an asymmetrically routed client loses name resolution. test_fragments built its guard list from a fixture of three entries while the role shipped six, so it validated the template against a port list nobody deploys. It now takes the shape from the declaration and fills only the numbers. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/08_networks.yml | 1 + roles/svc-net-tor/templates/nftables.conf.j2 | 9 +- roles/svc-net-tor/vars/main.yml | 13 ++ .../roles/svc-net-firewall/test_fragments.py | 37 ++++- .../roles/svc-net-tor/test_guarded_ports.py | 130 ++++++++++++++++++ 5 files changed, 178 insertions(+), 12 deletions(-) create mode 100644 tests/unit/python/roles/svc-net-tor/test_guarded_ports.py diff --git a/group_vars/all/08_networks.yml b/group_vars/all/08_networks.yml index c90530f319..05752e60bd 100644 --- a/group_vars/all/08_networks.yml +++ b/group_vars/all/08_networks.yml @@ -5,6 +5,7 @@ NETWORK_PUBLIC_DNS_RESOLVERS: - 8.8.8.8 # nocheck: hardcoded-dns-resolver - 1.0.0.1 # nocheck: hardcoded-dns-resolver NETWORK_DOCKER_EMBEDDED_DNS_RESOLVER: "127.0.0.11" # nocheck: hardcoded-dns-resolver +NETWORK_DNS_PORT: 53 # Well-known resolver port a host DNS listener binds NETWORK_DOCKER_DNS_NGINX_DIRECTIVE: "resolver {{ NETWORK_DOCKER_EMBEDDED_DNS_RESOLVER }} valid=30s ipv6={{ NETWORK_IPV6_ENABLED | on_off }};" NETWORK_LOOPBACK_CIDR: "127.0.0.0/8" NETWORK_PRIVATE_CIDRS: diff --git a/roles/svc-net-tor/templates/nftables.conf.j2 b/roles/svc-net-tor/templates/nftables.conf.j2 index 974ae97a63..eab3aa5bb0 100644 --- a/roles/svc-net-tor/templates/nftables.conf.j2 +++ b/roles/svc-net-tor/templates/nftables.conf.j2 @@ -2,25 +2,24 @@ table inet {{ FIREWALL_TABLE }} delete table inet {{ FIREWALL_TABLE }} table inet {{ FIREWALL_TABLE }} { - set admitted { + set containers { type ipv4_addr flags interval elements = { {{ TOR_EGRESS_CLIENT_CIDRS | join(', ') }} } } - set socks_admitted { + set internal { type ipv4_addr flags interval elements = { {{ NETWORK_INTERNAL_CIDRS | join(', ') }} } } -{% set _tcp_guarded = ((TOR_EGRESS_GUARDED_PORTS | selectattr('protocol', 'equalto', 'tcp') | map(attribute='port') | list) + [TOR_SOCKS_PORT]) | unique | list %} +{% set _tcp_guarded = TOR_EGRESS_GUARDED_PORTS | selectattr('protocol', 'equalto', 'tcp') | map(attribute='port') | unique | list %} {% set _udp_guarded = TOR_EGRESS_GUARDED_PORTS | selectattr('protocol', 'equalto', 'udp') | map(attribute='port') | unique | list %} chain guard { {% for entry in TOR_EGRESS_GUARDED_PORTS %} - ip saddr != @admitted {{ entry.protocol }} dport {{ entry.port }} drop + ip saddr != @{{ entry.admitted }} {{ entry.protocol }} dport {{ entry.port }} drop {% endfor %} - ip saddr != @socks_admitted tcp dport {{ TOR_SOCKS_PORT }} drop {% if _tcp_guarded | length > 0 %} fib saddr . iif oif missing tcp dport { {{ _tcp_guarded | join(', ') }} } drop {% endif %} diff --git a/roles/svc-net-tor/vars/main.yml b/roles/svc-net-tor/vars/main.yml index a02e9f2d67..d8b49547b7 100644 --- a/roles/svc-net-tor/vars/main.yml +++ b/roles/svc-net-tor/vars/main.yml @@ -12,15 +12,28 @@ TOR_FLAVOR: "{{ lookup('config', application_id, 'services.tor.flavor') }}" TOR_TRANS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.trans') }}" TOR_DNS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.dns') }}" TOR_SOCKS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.socks') }}" +TOR_DNSMASQ_PORT: "{{ NETWORK_DNS_PORT }}" TOR_DNSMASQ_CONF: "/etc/dnsmasq.d/tor-onion.conf" TOR_EGRESS_GUARDED_PORTS: - port: "{{ TOR_TRANS_PORT }}" protocol: tcp + admitted: containers - port: "{{ TOR_DNS_PORT }}" protocol: tcp + admitted: containers - port: "{{ TOR_DNS_PORT }}" protocol: udp + admitted: containers + - port: "{{ TOR_SOCKS_PORT }}" + protocol: tcp + admitted: internal + - port: "{{ TOR_DNSMASQ_PORT }}" + protocol: tcp + admitted: internal + - port: "{{ TOR_DNSMASQ_PORT }}" + protocol: udp + admitted: internal TOR_EGRESS_CLIENT_CIDRS: "{{ lookup('tor_egress_clients') }}" TOR_DNSMASQ_DROPIN: "/etc/systemd/system/dnsmasq.service.d/infinito-restart.conf" # nocheck: host-local-fact meta/services.yml pins this role to the manager, so the fact describes the node the hidden service forwards on diff --git a/tests/unit/python/roles/svc-net-firewall/test_fragments.py b/tests/unit/python/roles/svc-net-firewall/test_fragments.py index bc7828ab8f..99152196b5 100644 --- a/tests/unit/python/roles/svc-net-firewall/test_fragments.py +++ b/tests/unit/python/roles/svc-net-firewall/test_fragments.py @@ -32,23 +32,46 @@ from pathlib import Path import jinja2 from utils.cache.files import iter_project_files, read_text +from utils.cache.yaml import load_yaml_any +from utils.roles.mapping import ROLE_FILE_VARS_MAIN from . import PROJECT_ROOT FRAGMENT = "templates/nftables.conf.j2" TABLE = "infinito_probe" +PROBE_PORTS = { + "TOR_TRANS_PORT": "9040", + "TOR_DNS_PORT": "9053", + "TOR_SOCKS_PORT": "9050", + "TOR_DNSMASQ_PORT": "53", +} + +_PORT_VARIABLE = re.compile(r"\{\{\s*([A-Z][A-Z0-9_]*)\s*\}\}") + + +def _guarded_ports() -> list[dict]: + """The role's own guard list, with its port variables filled from stand-ins. + + The shape has to come from the declaration: an entry that gains a key, a + protocol or a whole port must reach the rendered table here, or this check + keeps passing on a fixture the role stopped shipping. + """ + declared = load_yaml_any( + str(PROJECT_ROOT / "roles" / "svc-net-tor" / ROLE_FILE_VARS_MAIN) + )["TOR_EGRESS_GUARDED_PORTS"] + return [ + {**entry, "port": PROBE_PORTS[_PORT_VARIABLE.search(entry["port"]).group(1)]} + for entry in declared + ] + + COMMON = { "FIREWALL_TABLE": TABLE, "TOR_EGRESS_CLIENT_CIDRS": ["127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"], - "TOR_EGRESS_GUARDED_PORTS": [ - {"port": "9040", "protocol": "tcp"}, - {"port": "9053", "protocol": "tcp"}, - {"port": "9053", "protocol": "udp"}, - ], + "TOR_EGRESS_GUARDED_PORTS": _guarded_ports(), "TOR_EGRESS_VIRTUAL_NET_IPV4": "10.192.0.0/10", - "TOR_TRANS_PORT": "9040", - "TOR_SOCKS_PORT": "9050", + **PROBE_PORTS, "TOR_CONTAINER_DNS_HOST": "172.17.0.1", "NETWORK_INTERNAL_CIDRS": [ "127.0.0.0/8", diff --git a/tests/unit/python/roles/svc-net-tor/test_guarded_ports.py b/tests/unit/python/roles/svc-net-tor/test_guarded_ports.py new file mode 100644 index 0000000000..f3c2c0ba88 --- /dev/null +++ b/tests/unit/python/roles/svc-net-tor/test_guarded_ports.py @@ -0,0 +1,130 @@ +"""Every port the egress router binds locally is dropped for foreign sources. + +The guard chain is the only thing between these listeners and anyone who can +route a packet to the node. A port that is bound but not named in a drop rule +answers the whole reachable network: the transparent proxy relays, the DNSPort +resolves, and the split-DNS forwarder both resolves recursively and hands out +the node's own ``.onion`` records. + +This walks the real declaration chain rather than a fixture: ``meta/services.yml`` +names the ports tor itself binds, ``vars/main.yml`` turns each into a +``TOR_*_PORT`` variable, and the guard list plus the fragment template spend +those variables on drop rules. A port added to the service declaration without a +rule fails the first test. + +The forwarder sits outside that chain because it is not tor's own port: it takes +the well-known resolver port from ``NETWORK_DNS_PORT``, and the host already +assigns that number to whichever resolver owns it. The second test covers it by +name instead. +""" + +from __future__ import annotations + +import re +import unittest + +from utils.cache.files import read_text +from utils.cache.yaml import load_yaml_any +from utils.roles.mapping import ROLE_FILE_META_SERVICES, ROLE_FILE_VARS_MAIN + +from . import PROJECT_ROOT + +ROLE = PROJECT_ROOT / "roles/svc-net-tor" +SERVICES = ROLE / ROLE_FILE_META_SERVICES +VARS = ROLE / ROLE_FILE_VARS_MAIN +FRAGMENT = ROLE / "templates/nftables.conf.j2" + +_VARIABLE = re.compile(r"\{\{\s*([A-Z][A-Z0-9_]*)\s*\}\}") +_PORT_KEY = re.compile(r"services\.tor\.ports\.local\.([a-z0-9_]+)") +_DROP_RULE = re.compile(r"^\s*(?!#).*\b(?Ptcp|udp)\s+dport\b.*\bdrop\b") + + +def _declared_ports() -> dict[str, int]: + return load_yaml_any(str(SERVICES))["tor"]["ports"]["local"] + + +def _port_variables() -> dict[str, str]: + """Maps each ``TOR_*_PORT`` variable to the port key it reads.""" + declared = load_yaml_any(str(VARS)) + names = {} + for name, value in declared.items(): + if not isinstance(value, str): + continue + key = _PORT_KEY.search(value) + if key is not None: + names[name] = key.group(1) + return names + + +def _entries_for(variable: str) -> list[dict]: + """Guard entries whose port comes from *variable*.""" + return [ + entry + for entry in load_yaml_any(str(VARS))["TOR_EGRESS_GUARDED_PORTS"] + if _VARIABLE.search(str(entry["port"])).group(1) == variable + ] + + +def _guarded_keys() -> set[str]: + """Port keys that reach a drop rule, through the vars list or the template.""" + variables = _port_variables() + guarded: set[str] = set() + + for entry in load_yaml_any(str(VARS))["TOR_EGRESS_GUARDED_PORTS"]: + name = _VARIABLE.search(str(entry["port"])) + if name is not None and name.group(1) in variables: + guarded.add(variables[name.group(1)]) + + for line in read_text(str(FRAGMENT)).splitlines(): + if _DROP_RULE.match(line) is None: + continue + for name in _VARIABLE.findall(line): + if name in variables: + guarded.add(variables[name]) + + return guarded + + +class TestGuardedPorts(unittest.TestCase): + def test_every_locally_bound_port_reaches_a_drop_rule(self) -> None: + declared = _declared_ports() + self.assertTrue(declared, f"{SERVICES} declares no local ports") + + unguarded = sorted(set(declared) - _guarded_keys()) + self.assertEqual( + unguarded, + [], + "these ports are bound on the node but no guard rule drops a " + "foreign source for them, so they answer everything that can route " + f"to the node: {', '.join(f'{key} ({declared[key]})' for key in unguarded)}", + ) + + def test_the_forwarder_is_guarded_on_both_transports(self) -> None: + entries = _entries_for("TOR_DNSMASQ_PORT") + self.assertEqual( + {entry["protocol"] for entry in entries}, + {"tcp", "udp"}, + "dnsmasq answers on both transports, and a TCP query returns the " + "same records a UDP one does", + ) + + def test_the_forwarder_admits_every_source_the_node_calls_internal(self) -> None: + """The resolver may not be stricter than the proxy it sits next to. + + ``internal`` is ``NETWORK_INTERNAL_CIDRS``, the same list ``torrc.j2`` + spends on ``SocksPolicy accept``. ``containers`` is narrower: out of + ``10.0.0.0/8`` it carries only the docker pool and the exception role, + which leaves out swarm's ingress network and every overlay docker + allocates without an ipam block. Guarding the resolver with the narrow + set hands a source a full SOCKS proxy and refuses it a name lookup. + """ + self.assertEqual( + {entry["admitted"] for entry in _entries_for("TOR_DNSMASQ_PORT")}, + {"internal"}, + "the resolver carries ordinary name resolution for everything the " + "node routes, so it takes the internal set, not the container pools", + ) + + +if __name__ == "__main__": + unittest.main() From b12ff8d0c411f5b935368971bc56aafe401f3c2b Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 01:27:04 +0200 Subject: [PATCH 058/197] fix(svc-net-tor): tie the dnsmasq reload to the branch that writes its drop-in The systemd drop-in is written inside a block gated on TOR_DNSMASQ_OWNS_LISTENER, which turns false as soon as a foreign file sits in /etc/dnsmasq.d. The start task read that block's register unconditionally, so the expression's correctness depended on ansible defining a register for a task it skipped. Short-circuiting on the same condition means the register is never reached when the block did not run, and the question stops mattering. The test asserted the literal declaration string, which exercises nothing. It now renders the expression read from the role with ansible's own bool filter and a changed test, in both states: owning the listener with a rewritten drop-in yields True, not owning it yields False without touching the register. The previous form raises UndefinedError under the second case. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-net-tor/tasks/router.yml | 2 +- .../svc-net-tor/test_container_dns_wiring.py | 20 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/roles/svc-net-tor/tasks/router.yml b/roles/svc-net-tor/tasks/router.yml index 7d400f9521..08f4119e8e 100644 --- a/roles/svc-net-tor/tasks/router.yml +++ b/roles/svc-net-tor/tasks/router.yml @@ -68,7 +68,7 @@ ansible.builtin.systemd: name: dnsmasq enabled: true - daemon_reload: "{{ _tor_dnsmasq_dropin is changed }}" + daemon_reload: "{{ TOR_DNSMASQ_OWNS_LISTENER | bool and _tor_dnsmasq_dropin is changed }}" state: "{{ 'restarted' if (_tor_dnsmasq_conf is changed or _tor_dnsmasq_confdir is changed) else 'started' }}" - name: "๐Ÿ”€ Egress router | Route host name resolution through dnsmasq" when: TOR_DNSMASQ_OWNS_LISTENER | bool diff --git a/tests/integration/roles/svc-net-tor/test_container_dns_wiring.py b/tests/integration/roles/svc-net-tor/test_container_dns_wiring.py index 7f0dc0d3c7..4ed72e9dd0 100644 --- a/tests/integration/roles/svc-net-tor/test_container_dns_wiring.py +++ b/tests/integration/roles/svc-net-tor/test_container_dns_wiring.py @@ -1,5 +1,6 @@ import unittest +from ansible.module_utils.parsing.convert_bool import boolean as ansible_boolean from jinja2 import Environment, FileSystemLoader from tests.integration.roles import PROJECT_ROOT @@ -143,9 +144,24 @@ class TestStaticBindSurvivesBoot(unittest.TestCase): names = [t.get("name") for t in tasks] start = next(t for t in tasks if t.get("name") == START_TASK) self.assertLess(names.index(DROPIN_BLOCK), names.index(START_TASK)) + + reload_expression = start["ansible.builtin.systemd"]["daemon_reload"] + env = Environment(autoescape=False) # noqa: S701 - renders a boolean, not markup + env.filters["bool"] = ansible_boolean + env.tests["changed"] = lambda result: bool(result.get("changed", False)) + self.assertEqual( - start["ansible.builtin.systemd"]["daemon_reload"], - "{{ _tor_dnsmasq_dropin is changed }}", + env.from_string(reload_expression).render( + TOR_DNSMASQ_OWNS_LISTENER=True, _tor_dnsmasq_dropin={"changed": True} + ), + "True", + "a rewritten drop-in has to reach systemd before the unit restarts", + ) + self.assertEqual( + env.from_string(reload_expression).render(TOR_DNSMASQ_OWNS_LISTENER=False), + "False", + "the drop-in task is gated on owning the listener, so the reload " + "expression must not reach for its register when it was skipped", ) def test_the_bridge_fact_is_live_before_anything_binds_it(self): From 351645ef272055e53a28b990b4bb91c7a97e5d74 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 02:05:39 +0200 Subject: [PATCH 059/197] fix(make): let swarm-zombie's discovery emit the variant act asks for The target decides its matrix from two ends. ACT_MATRIX picks from the finished rows, ACT_INPUTS decides which rows exist at all. 86033b63f1 drew that distinction for the mode axis and pinned modes=swarm; the variant axis stayed open, so the whitelist reached discovery unpinned and only variant 0 was ever produced. With variant=1 the intersection was therefore empty: act found no matrix entry for the deploy job, ran discovery alone and exited 0 - the same green-while-drilling-nothing shape 9e06f4c9a6 set out to refuse. Its guard did not catch this one because it validates that the selection token names a real row, not that discovery will emit it under the inputs the recipe passes. Measured: before the change, 'make swarm-zombie app=svc-net-tor variant=1' ended after the discovery job with a plan naming variant 0 and no deploy; after it, the same invocation runs the deploy job for that row. Co-Authored-By: Claude Opus 5 (1M context) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3845d55688..b743565ca4 100644 --- a/Makefile +++ b/Makefile @@ -808,7 +808,7 @@ swarm-zombie: install-act ACT_WORKFLOW=.github/workflows/call-test-deploy.yml \ ACT_JOB=deploy \ ACT_MATRIX="apps:$(app);variant:$(or $(variant),0);mode:swarm" \ - ACT_INPUTS="whitelist=$(app) distros=$(SWARM_DISTROS) index=0 sweep=0 modes=swarm disable=$(disable)" \ + ACT_INPUTS="whitelist=$(app)#$(or $(variant),0)@swarm distros=$(SWARM_DISTROS) index=0 sweep=0 modes=swarm disable=$(disable)" \ bash scripts/tests/deploy/act/workflow.sh .PHONY: system-purge From e74a58540209c4a0bb92d77980d09b4810794300 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 02:17:26 +0200 Subject: [PATCH 060/197] fix(guide): ask the reader for the clearnet resolver a Tor node needs svc-net-tor aborts at roles/svc-net-tor/tasks/router.yml:2 when networks.internet.dns is empty, and rightly so: dnsmasq runs with no-resolv, so an empty upstream leaves the node answering .onion and nothing else. That value is set in exactly two places, both under inventories/development/, and the documented production path reaches neither - the provisioner writes no networks key at all. A reader following the role README verbatim therefore cannot deploy a Tor node. CI has been proving this since 2026-08-31 without anyone seeing it: the README replay rides on the role's smallest variant and is skipped when the deploy is red, so the only row that ever reached it is one where svc-net-tor itself went green. That happened first on Ubuntu, where the local-service defect abb6f37297 fixes never fired. The template now emits the DNS assignment, the -e passthrough and the networks entry in --vars for a role whose tasks name networks.internet.dns, which today is svc-net-tor alone; the generator derives that rather than taking a declaration, so a second role asking for the value documents it by doing so. scripts/github/guide/compose_deploy.sh substitutes the placeholder from INFINITO_DNS_IP, the same SPOT inventories/development/default.yml reads, next to the existing domain substitution. This follows 972596a758, which added the administrator key to the same block for the same reason. No fallback inside the role: defaulting to NETWORK_PUBLIC_DNS_RESOLVERS would route a Tor node's clearnet queries to Cloudflare without the operator choosing it, which is what the assert exists to prevent. Co-Authored-By: Claude Opus 5 (1M context) --- cli/build/docs/readme/generate.py | 14 ++++++++++++++ roles/svc-net-tor/README.md | 5 +++-- scripts/github/guide/compose_deploy.sh | 1 + templates/roles/README.md.j2.tmpl | 12 +++++++++--- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/cli/build/docs/readme/generate.py b/cli/build/docs/readme/generate.py index 40a76041ab..201362567f 100644 --- a/cli/build/docs/readme/generate.py +++ b/cli/build/docs/readme/generate.py @@ -70,6 +70,19 @@ def _app_name(preamble: str, role_name: str) -> str: return entity.replace("-", " ").title() +def _needs_clearnet_resolver(role_dir) -> bool: + """Whether the role refuses to deploy without ``networks.internet.dns``. + + Only the inventories under ``inventories/development/`` set that value, so a + role asserting it has to name it in the block an operator copies, or the + documented deploy aborts on the assert. + """ + return any( + "networks.internet.dns" in read_text(str(path)) + for path in sorted((role_dir / "tasks").rglob("*.yml")) + ) + + def _base_context(role_dir, role_name: str, app_name: str, *, invokable: bool) -> dict: return { "application_id": role_name, @@ -84,6 +97,7 @@ def _base_context(role_dir, role_name: str, app_name: str, *, invokable: bool) - "application_author": _role_author(role_dir), "application_author_url": author_urls().get(_role_author(role_dir)), "application_is_host": not role_has_stack(role_dir), + "application_needs_clearnet_resolver": _needs_clearnet_resolver(role_dir), "cosmos_mermaid": derive_cosmos_mermaid(role_dir, role_name), "cosmos_legend": _cosmos_legend(), } diff --git a/roles/svc-net-tor/README.md b/roles/svc-net-tor/README.md index 0dfdf7a21c..1091186daf 100644 --- a/roles/svc-net-tor/README.md +++ b/roles/svc-net-tor/README.md @@ -86,19 +86,20 @@ Run the published image to provision the inventory and deploy Tor Onion Service APP=svc-net-tor HOST= DOMAIN= +DNS= TLS_MODE=self_signed SSH_PUBLIC_KEY="" docker run --rm -it \ -v "$PWD/inventories:/etc/infinito.nexus/inventories" \ - -e APP="$APP" -e HOST="$HOST" -e DOMAIN="$DOMAIN" -e TLS_MODE="$TLS_MODE" -e SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" \ + -e APP="$APP" -e HOST="$HOST" -e DOMAIN="$DOMAIN" -e DNS="$DNS" -e TLS_MODE="$TLS_MODE" -e SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" \ ghcr.io/infinito-nexus/core/debian bash -c ' INVENTORY=/etc/infinito.nexus/inventories/production infinito administration inventory provision "$INVENTORY" \ --inventory-file "$INVENTORY/devices.yml" \ --host "$HOST" \ --include "$APP" \ - --vars "{\"TLS_MODE\": \"$TLS_MODE\", \"DOMAIN_PRIMARY\": \"$DOMAIN\", \"users\": {\"administrator\": {\"authorized_keys\": [\"$SSH_PUBLIC_KEY\"]}}}" && + --vars "{\"TLS_MODE\": \"$TLS_MODE\", \"DOMAIN_PRIMARY\": \"$DOMAIN\", \"networks\": {\"internet\": {\"dns\": \"$DNS\"}}, \"users\": {\"administrator\": {\"authorized_keys\": [\"$SSH_PUBLIC_KEY\"]}}}" && infinito administration deploy dedicated "$INVENTORY/devices.yml" \ --password-file "$INVENTORY/.password" \ --diff -vv' diff --git a/scripts/github/guide/compose_deploy.sh b/scripts/github/guide/compose_deploy.sh index b7fdcc16d8..ff6a82fd9f 100755 --- a/scripts/github/guide/compose_deploy.sh +++ b/scripts/github/guide/compose_deploy.sh @@ -16,4 +16,5 @@ test -s /tmp/run.sh source scripts/meta/env/load.sh sed -i "s##ssh-ed25519 AAAA_TEST_DUMMY_KEY github-ci-dummy@infinito#" /tmp/run.sh sed -i "s##${INFINITO_DOMAIN}#" /tmp/run.sh +sed -i "s##${INFINITO_DNS_IP}#" /tmp/run.sh docker exec -i "${INFINITO_CONTAINER}" bash -s +{% if application_needs_clearnet_resolver -%} +DNS= +{% endif -%} TLS_MODE=self_signed SSH_PUBLIC_KEY="" INVENTORY=inventories/production @@ -91,7 +94,7 @@ infinito administration inventory provision "$INVENTORY" \ --inventory-file "$INVENTORY/devices.yml" \ --host localhost \ --include "$APP" \ - --vars "{\"TLS_MODE\": \"$TLS_MODE\", \"DOMAIN_PRIMARY\": \"$DOMAIN\", \"users\": {\"administrator\": {\"authorized_keys\": [\"$SSH_PUBLIC_KEY\"]}}}" + --vars "{\"TLS_MODE\": \"$TLS_MODE\", \"DOMAIN_PRIMARY\": \"$DOMAIN\", {% if application_needs_clearnet_resolver %}\"networks\": {\"internet\": {\"dns\": \"$DNS\"}}, {% endif %}\"users\": {\"administrator\": {\"authorized_keys\": [\"$SSH_PUBLIC_KEY\"]}}}" infinito administration deploy dedicated "$INVENTORY/devices.yml" \ --password-file "$INVENTORY/.password" \ --diff -vv @@ -103,19 +106,22 @@ Run the published image to provision the inventory and deploy {{ application_nam APP={{ application_id }} HOST= DOMAIN= +{% if application_needs_clearnet_resolver -%} +DNS= +{% endif -%} TLS_MODE=self_signed SSH_PUBLIC_KEY="" docker run --rm -it \ -v "$PWD/inventories:/etc/infinito.nexus/inventories" \ - -e APP="$APP" -e HOST="$HOST" -e DOMAIN="$DOMAIN" -e TLS_MODE="$TLS_MODE" -e SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" \ + -e APP="$APP" -e HOST="$HOST" -e DOMAIN="$DOMAIN" {% if application_needs_clearnet_resolver %}-e DNS="$DNS" {% endif %}-e TLS_MODE="$TLS_MODE" -e SSH_PUBLIC_KEY="$SSH_PUBLIC_KEY" \ ghcr.io/infinito-nexus/core/debian bash -c ' INVENTORY=/etc/infinito.nexus/inventories/production infinito administration inventory provision "$INVENTORY" \ --inventory-file "$INVENTORY/devices.yml" \ --host "$HOST" \ --include "$APP" \ - --vars "{\"TLS_MODE\": \"$TLS_MODE\", \"DOMAIN_PRIMARY\": \"$DOMAIN\", \"users\": {\"administrator\": {\"authorized_keys\": [\"$SSH_PUBLIC_KEY\"]}}}" && + --vars "{\"TLS_MODE\": \"$TLS_MODE\", \"DOMAIN_PRIMARY\": \"$DOMAIN\", {% if application_needs_clearnet_resolver %}\"networks\": {\"internet\": {\"dns\": \"$DNS\"}}, {% endif %}\"users\": {\"administrator\": {\"authorized_keys\": [\"$SSH_PUBLIC_KEY\"]}}}" && infinito administration deploy dedicated "$INVENTORY/devices.yml" \ --password-file "$INVENTORY/.password" \ --diff -vv' From 2f533eb27952cbf5038ec14be3cc94b984ed5c1f Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 08:54:01 +0200 Subject: [PATCH 061/197] refactor(svc-net-tor): declare the role's config in the role, and the onion's HTTP forward at all group_vars/all/README.md forbids role-specific configuration in that directory, and 19_tor.yml held five variables no role but svc-net-tor ever read. They now live where the README points: the values an operator may override in meta/services.yml, where the inventory's applications block is deep-merged on top, and the derived ones in vars/main.yml. The exception is the egress virtual range, which moves the other way into 08_networks.yml. It is not a property of the role but a claim on shared address space, and 0f6728a4c1 records what a collision costs: tor's upstream default 10.192.0.0/10 swallowed the docker pool at 10.208.0.0/12, and every connection to a container address came back as 'errno 104 Connection reset by peer'. The two numbers that must not overlap now sit one line apart. The onion's HTTP forward becomes a declaration rather than a derivation. lookup('tor_ports') walks group_names and collects what those roles declare under ports.onion; port 80 belongs to svc-prx-openresty, which a node acquires through sys-stk-front-proxy and which therefore appears in no dependency list an inventory carries. A node provisioned with --include svc-net-tor published an onion forwarding SSH and nothing else, leaving every app it serves unreachable - measured as ERR_SOCKS_CONNECTION_FAILED in the CI instructions replay. The new role-local tor_extra_ports lookup composes the flagged forwards with the derived ones and collapses a port named twice, since torrc writes one HiddenServicePort line per entry. Not moved: tor_ports stays in plugins/lookup because tor_reachable imports from it and three database roles use that; and the flags sit outside ports.onion because every key there mirrors a port the role declares under ports.local, which neither sshd nor the front proxy is. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/08_networks.yml | 1 + group_vars/all/19_tor.yml | 7 -- plugins/lookup/tor_ports.py | 4 +- .../lookup_plugins/tor_extra_ports.py | 80 +++++++++++++ roles/svc-net-tor/meta/services.yml | 5 + roles/svc-net-tor/vars/main.yml | 5 + .../roles/svc-net-tor/test_extra_ports.py | 105 ++++++++++++++++++ 7 files changed, 198 insertions(+), 9 deletions(-) delete mode 100644 group_vars/all/19_tor.yml create mode 100644 roles/svc-net-tor/lookup_plugins/tor_extra_ports.py create mode 100644 tests/unit/python/roles/svc-net-tor/test_extra_ports.py diff --git a/group_vars/all/08_networks.yml b/group_vars/all/08_networks.yml index 05752e60bd..f757e83a3e 100644 --- a/group_vars/all/08_networks.yml +++ b/group_vars/all/08_networks.yml @@ -21,6 +21,7 @@ NETWORK_TRUSTED_PROXY_CIDRS: "{{ ['127.0.0.1', '::1'] + (NETWORK_PRIVATE_CIDRS.v NETWORK_DOCKER_ADDRESS_POOLS: - base: "10.208.0.0/12" size: 24 +NETWORK_TOR_VIRTUAL_CIDR: "10.240.0.0/12" # Range tor answers for .onion instead of routing; must not overlap the pool above NETWORK_CONTAINER_BRIDGE_RESOLVER: false NETWORK_NESTED_DOCKER_BIP: "192.168.67.1/24" NETWORK_NESTED_DOCKER_ADDRESS_POOLS: diff --git a/group_vars/all/19_tor.yml b/group_vars/all/19_tor.yml deleted file mode 100644 index 691a0e1a99..0000000000 --- a/group_vars/all/19_tor.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -TOR_ONION_SSH_ENABLED: true - -TOR_EGRESS_ROUTER_ENABLED: true -TOR_EGRESS_VIRTUAL_NET_IPV4: "10.240.0.0/12" - -TOR_ONION_EXTRA_PORTS: "{{ ([{'onion_port': 22, 'target': '127.0.0.1:22'}] if (TOR_ONION_SSH_ENABLED | bool) else []) + lookup('tor_ports') }}" diff --git a/plugins/lookup/tor_ports.py b/plugins/lookup/tor_ports.py index 68f2bd9413..fea32575f8 100644 --- a/plugins/lookup/tor_ports.py +++ b/plugins/lookup/tor_ports.py @@ -143,8 +143,8 @@ class LookupModule(LookupBase): Both are read from the variant-merged applications view, so a variant that drops the opt-in drops the forward with it. The published port binds the host interface, so the loopback target reaches it from svc-net-tor's - host-network container. Consumed by ``TOR_ONION_EXTRA_PORTS`` - (group_vars/all/19_tor.yml) and rendered as ``HiddenServicePort`` lines in + host-network container. Composed with the flagged forwards by + ``lookup('tor_extra_ports')`` and rendered as ``HiddenServicePort`` lines in svc-net-tor's torrc. """ diff --git a/roles/svc-net-tor/lookup_plugins/tor_extra_ports.py b/roles/svc-net-tor/lookup_plugins/tor_extra_ports.py new file mode 100644 index 0000000000..db1b84bb45 --- /dev/null +++ b/roles/svc-net-tor/lookup_plugins/tor_extra_ports.py @@ -0,0 +1,80 @@ +"""Lookup `tor_extra_ports`: every port the node onion forwards. + +Two of them are flags rather than derivations, because no inventory carries +them. ``lookup('tor_ports')`` walks ``group_names`` and collects what the roles +listed there declare under ``ports.onion``; SSH belongs to no role at all, and +HTTP belongs to ``svc-prx-openresty``, which a node acquires through +``sys-stk-front-proxy`` and which therefore appears in no dependency list an +inventory holds. A node provisioned with ``--include svc-net-tor`` alone +published an onion forwarding SSH and nothing else, leaving every app it serves +unreachable over that onion. + +Forwarding HTTP is what this role exists for, so it is declared, and a node that +must not answer HTTP over its onion turns ``TOR_ONION_HTTP_ENABLED`` off. + +The derived ports still come from ``tor_ports``; a port named twice collapses, +since ``torrc.j2`` writes one ``HiddenServicePort`` line per entry. + +Usage: + + {{ lookup('tor_extra_ports') }} +""" + +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleError +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.plugins.loader import lookup_loader +from ansible.plugins.lookup import LookupBase + +FLAGGED_PORTS: tuple[tuple[str, int], ...] = ( + ("TOR_ONION_SSH_ENABLED", 22), + ("TOR_ONION_HTTP_ENABLED", 80), +) + + +def _entry(port: int) -> dict[str, Any]: + """One forward, in the shape ``torrc.j2`` consumes. + + Args: + port: the port the onion answers on; the backend host is substituted by + the template, so the target carries the port alone. + """ + return {"onion_port": port, "target": f"127.0.0.1:{port}"} + + +class LookupModule(LookupBase): + def run( + self, + terms: list[Any] | None, + variables: dict[str, Any] | None = None, + **kwargs: Any, + ) -> list[Any]: + if terms: + raise AnsibleError("lookup('tor_extra_ports') expects no terms.") + + variables = variables or getattr(self._templar, "available_variables", {}) or {} + + entries: list[dict[str, Any]] = [] + for name, port in FLAGGED_PORTS: + if name not in variables: + raise AnsibleError(f"lookup('tor_extra_ports'): {name} is not defined") + if boolean(variables[name]): + entries.append(_entry(port)) + + derived = lookup_loader.get( + "tor_ports", + loader=getattr(self, "_loader", None), + templar=getattr(self, "_templar", None), + ).run([], variables=variables, **kwargs)[0] + + seen = {entry["onion_port"] for entry in entries} + for entry in derived: + if entry["onion_port"] in seen: + continue + seen.add(entry["onion_port"]) + entries.append(entry) + + return [entries] diff --git a/roles/svc-net-tor/meta/services.yml b/roles/svc-net-tor/meta/services.yml index c28138c505..bf18fcf11d 100644 --- a/roles/svc-net-tor/meta/services.yml +++ b/roles/svc-net-tor/meta/services.yml @@ -26,6 +26,11 @@ tor: # nocheck: category-modes node: "" use_entry_guards: true flavor: public + forwards: + ssh: true + http: true + egress: + enabled: true timeout_multipliers: chutney: 2 public: 5 diff --git a/roles/svc-net-tor/vars/main.yml b/roles/svc-net-tor/vars/main.yml index d8b49547b7..f8245d1817 100644 --- a/roles/svc-net-tor/vars/main.yml +++ b/roles/svc-net-tor/vars/main.yml @@ -9,6 +9,11 @@ TOR_CONTAINER: "{{ lookup('config', application_id, 'services.tor.name') }}" TOR_NODE_ONION: "{{ lookup('config', application_id, 'services.tor.node', '') }}" TOR_USE_ENTRY_GUARDS: "{{ lookup('config', application_id, 'services.tor.use_entry_guards') | bool }}" TOR_FLAVOR: "{{ lookup('config', application_id, 'services.tor.flavor') }}" +TOR_ONION_SSH_ENABLED: "{{ lookup('config', application_id, 'services.tor.forwards.ssh') | bool }}" +TOR_ONION_HTTP_ENABLED: "{{ lookup('config', application_id, 'services.tor.forwards.http') | bool }}" +TOR_ONION_EXTRA_PORTS: "{{ lookup('tor_extra_ports') }}" +TOR_EGRESS_ROUTER_ENABLED: "{{ lookup('config', application_id, 'services.tor.egress.enabled') | bool }}" +TOR_EGRESS_VIRTUAL_NET_IPV4: "{{ NETWORK_TOR_VIRTUAL_CIDR }}" TOR_TRANS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.trans') }}" TOR_DNS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.dns') }}" TOR_SOCKS_PORT: "{{ lookup('config', application_id, 'services.tor.ports.local.socks') }}" diff --git a/tests/unit/python/roles/svc-net-tor/test_extra_ports.py b/tests/unit/python/roles/svc-net-tor/test_extra_ports.py new file mode 100644 index 0000000000..269c674a18 --- /dev/null +++ b/tests/unit/python/roles/svc-net-tor/test_extra_ports.py @@ -0,0 +1,105 @@ +"""The node onion forwards HTTP whether or not the proxy is in the inventory. + +``lookup('tor_ports')`` collects what the roles in ``group_names`` declare under +``ports.onion``. Port 80 belongs to ``svc-prx-openresty``, which a node acquires +through ``sys-stk-front-proxy`` and which therefore appears in no dependency +list an inventory holds. A node provisioned with ``--include svc-net-tor`` alone +published an onion forwarding SSH and nothing else. + +These run the real lookup over a stubbed ``tor_ports``, so a flag that stops +reaching the forward list, or a derived port that starts duplicating a flagged +one, fails here rather than on the node. +""" + +from __future__ import annotations + +import importlib.util +import sys +import unittest +from typing import Any +from unittest import mock + +from ansible.errors import AnsibleError + +from . import PROJECT_ROOT + +LOOKUP = "roles/svc-net-tor/lookup_plugins/tor_extra_ports.py" +SSH_ENTRY = {"onion_port": 22, "target": "127.0.0.1:22"} +HTTP_ENTRY = {"onion_port": 80, "target": "127.0.0.1:80"} +ENABLED = {"TOR_ONION_SSH_ENABLED": True, "TOR_ONION_HTTP_ENABLED": True} + + +def _load_lookup(): + path = PROJECT_ROOT / LOOKUP + spec = importlib.util.spec_from_file_location("tor_extra_ports_lookup", path) + module = importlib.util.module_from_spec(spec) + assert spec and spec.loader + sys.modules["tor_extra_ports_lookup"] = module + spec.loader.exec_module(module) + return module + + +class _DerivedPorts: + """Stands in for the tor_ports lookup the plugin composes with.""" + + def __init__(self, entries: list[dict[str, Any]]): + self.entries = entries + + def run(self, _terms, **_kwargs): + return [self.entries] + + +class TestExtraPorts(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.module = _load_lookup() + + def _ports(self, derived: list[dict], **variables) -> list[dict]: + lookup = self.module.LookupModule() + lookup._templar = None # noqa: SLF001 - the plugin reads variables we pass + lookup._loader = None # noqa: SLF001 + with mock.patch.object( + self.module.lookup_loader, "get", return_value=_DerivedPorts(derived) + ): + return lookup.run([], variables={**ENABLED, **variables})[0] + + def test_http_is_forwarded_without_the_proxy_in_the_inventory(self) -> None: + self.assertIn( + HTTP_ENTRY, + self._ports([]), + "a node whose inventory lists only svc-net-tor still serves its apps " + "over the onion, so the forward cannot depend on the proxy being listed", + ) + + def test_a_derived_port_does_not_duplicate_a_flagged_one(self) -> None: + ports = self._ports([HTTP_ENTRY]) + self.assertEqual( + [entry for entry in ports if entry["onion_port"] == 80], + [HTTP_ENTRY], + "torrc writes one HiddenServicePort line per entry, so a duplicate " + "publishes the same forward twice", + ) + + def test_derived_ports_still_reach_the_forward_list(self) -> None: + onion_dns = {"onion_port": 9053, "target": "127.0.0.1:9053"} + self.assertIn( + onion_dns, + self._ports([onion_dns]), + "the flags add to what the services declare, they do not replace it", + ) + + def test_a_flag_turns_its_forward_off(self) -> None: + ports = self._ports([], TOR_ONION_HTTP_ENABLED=False) + self.assertNotIn(HTTP_ENTRY, ports) + self.assertIn(SSH_ENTRY, ports, "the other flag is untouched") + + def test_an_undeclared_flag_is_refused(self) -> None: + lookup = self.module.LookupModule() + lookup._templar = None # noqa: SLF001 + with self.assertRaises(AnsibleError) as raised: + lookup.run([], variables={"TOR_ONION_SSH_ENABLED": True}) + self.assertIn("TOR_ONION_HTTP_ENABLED", str(raised.exception)) + + +if __name__ == "__main__": + unittest.main() From 1647dfb5428d9dc0509637800464bf085e779256 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 08:54:53 +0200 Subject: [PATCH 062/197] docs(group_vars): point role config at the two homes roles actually have The rule named roles//config/main.yml, which no role has: f65f4d60fb migrated the last one, web-app-pihole, to meta/services.yml and did not carry the README with it. A reader following the sentence found nowhere to put the value. It now names the two real homes and the property that decides between them. meta/services.yml is overridable, because the inventory's applications block is deep-merged on top of it and lookup('config', ...) reads the result; vars/main.yml is not, because role vars outrank inventory variables. The clause about a value only one role reads today is the case that just went wrong: group_vars/all/19_tor.yml looked harmless for exactly that reason. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/group_vars/all/README.md b/group_vars/all/README.md index 2def1b4732..233e6be0f0 100644 --- a/group_vars/all/README.md +++ b/group_vars/all/README.md @@ -6,7 +6,10 @@ This directory contains the [Ansible](https://docs.ansible.com/ansible/latest/pl Variables in this directory MUST apply across all hosts and all roles. Typical examples are the software identity, deployment toggles, domain defaults, the central network and port registries, OIDC and LDAP endpoints, design tokens, and resource defaults. -Role-specific configuration MUST NOT live here. Such values belong in the corresponding role under `roles//config/main.yml` or `roles//vars/main.yml`. +Role-specific configuration MUST NOT live here, even when only one role reads it today. Such values belong in the corresponding role: + +- `roles//meta/services.yml` for anything an operator may override, because the inventory's `applications.` block is deep-merged on top of it and `lookup('config', application_id, '')` reads the result. +- `roles//vars/main.yml` for values derived from that configuration, and for those the role fixes. Role vars outrank inventory variables, so a value placed here cannot be overridden per host. ## File naming and load order ๐Ÿ”ข From 01f9f6ad13e515799717c33433ffa55402ec7121 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 10:40:43 +0200 Subject: [PATCH 063/197] fix(hlth-csp): honour a declared healthy status code outside the inventory The skip list gated its whole loop body on the group_names selection, so a role reaching a node through a runtime include_role never contributed its server.status_codes.default. web-svc-mirror arrives that way on every Tor node and declares 404 as healthy; on a node provisioned from the svc-net-tor README the probe read that 404 as unreachable and hlth-csp could not start. Measured in CI job 104622514568: the unit rendered without --skip-domain and the checker reported ERR_SOCKS_CONNECTION_FAILED on the mirror onion 26 times. Only the service-disabled half depends on the selection now. The widened branch can contribute at most 14 of 266 applications, and naming a domain no vhost serves is inert because script.py filters an nginx-enumerated list. Co-Authored-By: Claude Opus 5 (1M context) --- roles/sys-ctl-hlth-csp/lookup_plugins/csp_skip_domains.py | 8 +++----- .../lookup_plugins/test_csp_skip_domains.py | 8 ++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/roles/sys-ctl-hlth-csp/lookup_plugins/csp_skip_domains.py b/roles/sys-ctl-hlth-csp/lookup_plugins/csp_skip_domains.py index 08d344c073..04eee7c474 100644 --- a/roles/sys-ctl-hlth-csp/lookup_plugins/csp_skip_domains.py +++ b/roles/sys-ctl-hlth-csp/lookup_plugins/csp_skip_domains.py @@ -58,7 +58,7 @@ def _selection_from(group_names: Any) -> set[str]: class LookupModule(LookupBase): """Return domains the CSP probe should skip. - Skips canonical + alias domains of every selected application whose + Skips canonical + alias domains of every application whose ``server.status_codes.default`` declares any HTTP code >= 400 (e.g. federation-only roles that legitimately serve 4xx at ``/``). @@ -118,10 +118,8 @@ class LookupModule(LookupBase): skip: set[str] = set() for app_id in applications: - if selection and app_id not in selection: - continue - - skip |= self._service_disabled_domains(applications, app_id) + if not selection or app_id in selection: + skip |= self._service_disabled_domains(applications, app_id) default = get( applications, diff --git a/tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_skip_domains.py b/tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_skip_domains.py index 2ab1e503f5..80defa526d 100644 --- a/tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_skip_domains.py +++ b/tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_skip_domains.py @@ -74,7 +74,7 @@ class CspSkipDomainsLookupTests(unittest.TestCase): apps = {"web-app-foo": {"domains": {"canonical": ["foo.example.com"]}}} self.assertEqual(self._run(apps), []) - def test_group_names_selection_filters_apps(self): + def test_a_declared_4xx_is_honoured_outside_the_selection(self): apps = { "web-app-bridgy": { "domains": {"canonical": ["bridgy.example.com"]}, @@ -82,8 +82,8 @@ class CspSkipDomainsLookupTests(unittest.TestCase): "status_codes": {"default": [404]}, }, }, - "web-app-other": { - "domains": {"canonical": ["other.example.com"]}, + "web-svc-mirror": { + "domains": {"canonical": ["mirror.example.com"]}, "server": { "status_codes": {"default": [404]}, }, @@ -91,7 +91,7 @@ class CspSkipDomainsLookupTests(unittest.TestCase): } self.assertEqual( self._run(apps, group_names=["web-app-bridgy"]), - ["bridgy.example.com"], + ["bridgy.example.com", "mirror.example.com"], ) def test_group_names_csv_string_is_accepted(self): From df2fc5084a2dc8efa6401022e7634693f5422e7b Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 10:45:57 +0200 Subject: [PATCH 064/197] fix(lmstudio): recover a wedged model preload instead of retrying into it A preload that fails leaves the daemon unable to move bytes: measured in CI job 104622514450, attempt 1 reached 437.37 MB of 531.07 MB and died with 'Download failed: Timed-out', then all five retries sat at 0.00% for a full hour each until CONTAINER_EXEC_TIMEOUT killed the client. Six executions, 18796s, no progress. Killing the client does not touch the daemon that owns the transfer, so a plain retry cannot help. The rescue restarts the container, which a local probe showed restores a working transfer, then re-resolves the CID because swarm may reschedule the restarted task under a fresh id, and preloads once more. Pull and reap moved into one included unit so the rescue also covers the async path, where the failure surfaces at the reap rather than at the pull. retries drops from 5 to 1, since retries N yields N+1 executions. Co-Authored-By: Claude Opus 5 (1M context) --- .../svc-ai-lmstudio/tasks/01_manager_ops.yml | 40 ++++--------------- roles/svc-ai-lmstudio/tasks/utils/preload.yml | 33 +++++++++++++++ roles/svc-ai-lmstudio/tasks/utils/recover.yml | 28 +++++++++++++ 3 files changed, 68 insertions(+), 33 deletions(-) create mode 100644 roles/svc-ai-lmstudio/tasks/utils/preload.yml create mode 100644 roles/svc-ai-lmstudio/tasks/utils/recover.yml diff --git a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml index 60f97c71e8..24a691460d 100644 --- a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml +++ b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml @@ -6,36 +6,10 @@ resolve_output_prefix: LMSTUDIO resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" -- name: โฌ‡๏ธ Pre-pull LM Studio models - ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node - container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} lms get --yes --gguf {{ model.source | quote }} - register: lmstudio_pull_result - loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" - loop_control: - loop_var: model - label: "{{ model.alias }}" - # rationale: async; model downloads run overlapped, results reaped below before playwright - async: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" - poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" - retries: "{{ 0 if ASYNC_ENABLED | bool else 5 }}" - delay: 15 - until: > - (lmstudio_pull_result.rc | default(0)) == 0 or - ('already downloaded' in (lmstudio_pull_result.stdout | default(''))) - changed_when: false - failed_when: >- - (lmstudio_pull_result.failed | default(false)) - and 'already downloaded' not in ((lmstudio_pull_result.stdout | default('')) ~ (lmstudio_pull_result.stderr | default(''))) - delegate_to: "{{ LMSTUDIO_HOST_NODE }}" - -- name: "โณ Reap the LM Studio model pre-pulls" - ansible.builtin.include_tasks: - file: utils/async/reap.yml - apply: - delegate_to: "{{ LMSTUDIO_HOST_NODE }}" - vars: - async_reap_jobs: "{{ lmstudio_pull_result.results | default([]) }}" - async_reap_label: "LM Studio model pre-pull" - async_reap_tolerate: - - "already downloaded" - ASYNC_TIME: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME }}" +- name: "โฌ‡๏ธ Preload the LM Studio models" + block: + - name: "โฌ‡๏ธ Preload the LM Studio models (first pass)" + ansible.builtin.include_tasks: utils/preload.yml + rescue: + - name: "โ™ป๏ธ Recover the wedged transfer and preload again" + ansible.builtin.include_tasks: utils/recover.yml diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload.yml b/roles/svc-ai-lmstudio/tasks/utils/preload.yml new file mode 100644 index 0000000000..9c668a977a --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/preload.yml @@ -0,0 +1,33 @@ +--- +- name: โฌ‡๏ธ Pre-pull LM Studio models + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} lms get --yes --gguf {{ model.source | quote }} + register: lmstudio_pull_result + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.alias }}" + async: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" + poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" + retries: "{{ 0 if ASYNC_ENABLED | bool else 1 }}" + delay: 15 + until: > + (lmstudio_pull_result.rc | default(0)) == 0 or + ('already downloaded' in (lmstudio_pull_result.stdout | default(''))) + changed_when: false + failed_when: >- + (lmstudio_pull_result.failed | default(false)) + and 'already downloaded' not in ((lmstudio_pull_result.stdout | default('')) ~ (lmstudio_pull_result.stderr | default(''))) + delegate_to: "{{ LMSTUDIO_HOST_NODE }}" + +- name: "โณ Reap the LM Studio model pre-pulls" + ansible.builtin.include_tasks: + file: utils/async/reap.yml + apply: + delegate_to: "{{ LMSTUDIO_HOST_NODE }}" + vars: + async_reap_jobs: "{{ lmstudio_pull_result.results | default([]) }}" + async_reap_label: "LM Studio model pre-pull" + async_reap_tolerate: + - "already downloaded" + ASYNC_TIME: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME }}" diff --git a/roles/svc-ai-lmstudio/tasks/utils/recover.yml b/roles/svc-ai-lmstudio/tasks/utils/recover.yml new file mode 100644 index 0000000000..cea5918d25 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/recover.yml @@ -0,0 +1,28 @@ +--- +- name: "๐Ÿฉบ Dump the LM Studio transfer record" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} + cat /root/.lmstudio/.internal/single-downloads-info.json + register: lmstudio_transfer_record + changed_when: false + failed_when: false + delegate_to: "{{ LMSTUDIO_HOST_NODE }}" + +- name: "๐Ÿฉบ Report the LM Studio transfer record" + ansible.builtin.debug: + var: lmstudio_transfer_record.stdout + +- name: "โ™ป๏ธ Restart LM Studio to release the wedged transfer" + ansible.builtin.command: container restart {{ LMSTUDIO_LOCAL_CID }} + changed_when: true + delegate_to: "{{ LMSTUDIO_HOST_NODE }}" + +- name: "๐Ÿ” Re-resolve the {{ application_id }} hosting node + container id" + ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml') }}" + vars: + resolve_service_key: "{{ entity_name }}" + resolve_output_prefix: LMSTUDIO + resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" + +- name: "โฌ‡๏ธ Preload the LM Studio models (after the restart)" + ansible.builtin.include_tasks: utils/preload.yml From 42250f91ec97a4e415ad9c8e58d91a5d05b65a13 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 11:16:25 +0200 Subject: [PATCH 065/197] fix(lmstudio): justify the preload rescue the way the rescue lint requires tests/lint/ansible/structure/test_no_rescue_blocks.py forbids every rescue block in roles/ and tasks/ unless its first task carries a same-line '# nocheck: rescue-' comment, and make test runs it. The preload recovery added in df2fc5084a had none, so the gate failed with exactly one offender. The block meets the rule the lint documents rather than only its regex: it performs real in-play recovery, since a timed-out transfer leaves the daemon reporting 0.00% for every further pull and only restarting the container that owns it clears that, which the workflow-level capture cannot do; and it ends by preloading again, so a failure that recovery does not fix still re-raises. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-lmstudio/tasks/01_manager_ops.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml index 24a691460d..d2b87885d1 100644 --- a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml +++ b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml @@ -11,5 +11,5 @@ - name: "โฌ‡๏ธ Preload the LM Studio models (first pass)" ansible.builtin.include_tasks: utils/preload.yml rescue: - - name: "โ™ป๏ธ Recover the wedged transfer and preload again" + - name: "โ™ป๏ธ Recover the wedged transfer and preload again" # nocheck: rescue-recovery a timed-out transfer leaves the daemon reporting 0.00% for every further pull, and only restarting the container it owns clears that; the generic capture cannot restart anything, and the block ends by preloading again so an unrecovered failure still re-raises ansible.builtin.include_tasks: utils/recover.yml From 9af984f1a9c3278822948b7b45c40a1712e498e8 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 11:35:13 +0200 Subject: [PATCH 066/197] fix(lmstudio): restore the async rationale the preload task must carry tests/lint/ansible/roles/test_async_reason_documented.py requires a one-line '# rationale: async; ...' directly above every async key and offers no nocheck escape, because fire-and-forget is a deployment-wide decision that has to hold in swarm and stay playwright compatible. A comment audit removed the line while moving the task into utils/preload.yml, and make test went red on it. The restored line answers the question the lint asks, naming the reader rather than the reason for the async: the reap two tasks below reads every job back before the play continues, so no model is still downloading when playwright starts. The marker is not narration; the comment lint accepts rationale: by design. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-lmstudio/tasks/utils/preload.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload.yml b/roles/svc-ai-lmstudio/tasks/utils/preload.yml index 9c668a977a..a50c35ec9e 100644 --- a/roles/svc-ai-lmstudio/tasks/utils/preload.yml +++ b/roles/svc-ai-lmstudio/tasks/utils/preload.yml @@ -7,6 +7,7 @@ loop_control: loop_var: model label: "{{ model.alias }}" + # rationale: async; the reap below reads every job back before the play continues, so no model is still downloading when playwright starts async: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" retries: "{{ 0 if ASYNC_ENABLED | bool else 1 }}" From d6e175c7aa85d098264da9b1b4203a1e4f522a80 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 11:36:28 +0200 Subject: [PATCH 067/197] style(svc-net-tor): drop three noqa directives ruff no longer needs The tor extra-ports test carried '# noqa: SLF001' on the three lines that set lookup._templar and lookup._loader, which the test must do because the plugin reads the variables it is handed from them. Ruff reports no violation there any more, so RUF100 removed the directives as unused during make autoformat. A suppression that suppresses nothing asserts a conflict that does not exist and invites the next reader to treat the access as a problem. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/python/roles/svc-net-tor/test_extra_ports.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/python/roles/svc-net-tor/test_extra_ports.py b/tests/unit/python/roles/svc-net-tor/test_extra_ports.py index 269c674a18..2086e5ca12 100644 --- a/tests/unit/python/roles/svc-net-tor/test_extra_ports.py +++ b/tests/unit/python/roles/svc-net-tor/test_extra_ports.py @@ -56,8 +56,8 @@ class TestExtraPorts(unittest.TestCase): def _ports(self, derived: list[dict], **variables) -> list[dict]: lookup = self.module.LookupModule() - lookup._templar = None # noqa: SLF001 - the plugin reads variables we pass - lookup._loader = None # noqa: SLF001 + lookup._templar = None + lookup._loader = None with mock.patch.object( self.module.lookup_loader, "get", return_value=_DerivedPorts(derived) ): @@ -95,7 +95,7 @@ class TestExtraPorts(unittest.TestCase): def test_an_undeclared_flag_is_refused(self) -> None: lookup = self.module.LookupModule() - lookup._templar = None # noqa: SLF001 + lookup._templar = None with self.assertRaises(AnsibleError) as raised: lookup.run([], variables={"TOR_ONION_SSH_ENABLED": True}) self.assertIn("TOR_ONION_HTTP_ENABLED", str(raised.exception)) From 92eb58f38a97a221d45bed68e8826b29255936dd Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 12:09:54 +0200 Subject: [PATCH 068/197] fix(lmstudio): recover the preload only in compose, re-raise it in swarm playbook.yml:4 runs hosts: all under any_errors_fatal with no serial and no max_fail_percentage. When a task fails there, ansible/plugins/strategy/linear.py:329-333 marks every host that did not itself fail, while strategy/__init__.py:575-576 adds the failing host to _failed_hosts only when its next state is COMPLETE, which a rescue prevents. A rescued preload therefore leaves the stack host running alone through every later role while the workers and the nfs server are already marked failed. The recovery is kept where it was measured: the stall came from a compose row, container exec -i lmstudio by name rather than a swarm task id. Swarm now re-raises the original failure as the first rescue task, the shape roles/web-app-keycloak/tasks/03_manager_ops/updates.yml:28-31 already uses in mirror image, and the nocheck marker moves with it because test_no_rescue_blocks.py:82 reads only the first rescue task's line. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-lmstudio/tasks/01_manager_ops.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml index d2b87885d1..b588c180f2 100644 --- a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml +++ b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml @@ -11,5 +11,10 @@ - name: "โฌ‡๏ธ Preload the LM Studio models (first pass)" ansible.builtin.include_tasks: utils/preload.yml rescue: - - name: "โ™ป๏ธ Recover the wedged transfer and preload again" # nocheck: rescue-recovery a timed-out transfer leaves the daemon reporting 0.00% for every further pull, and only restarting the container it owns clears that; the generic capture cannot restart anything, and the block ends by preloading again so an unrecovered failure still re-raises + - name: "๐Ÿ Re-raise the preload failure unchanged in swarm" # nocheck: rescue-recovery a timed-out transfer leaves the daemon reporting 0.00% for every further pull, and only restarting the container it owns clears that, which the generic capture cannot do; swarm re-raises here instead, because under any_errors_fatal a rescued host keeps running while every other host is already marked failed, and compose ends the block by preloading again so an unrecovered failure still re-raises + ansible.builtin.fail: + msg: "{{ ansible_failed_result.msg | default('LM Studio preload failed') }}" + when: DEPLOYMENT_MODE == 'swarm' + + - name: "โ™ป๏ธ Recover the wedged transfer and preload again" ansible.builtin.include_tasks: utils/recover.yml From e1d513a3c06cc14375dcfa430fd4404afdb329b2 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 13:10:16 +0200 Subject: [PATCH 069/197] fix(swarm-matrix): bake a variant payload for every app the round deploys The swarm matrix resolved variant payloads over the planner's round_include while provisioning the closure derive_includes returns. The two agree until a variant switches its couplings off: the planner then resolves no prerequisites, round_include collapses to the primary, and every deployed dependency silently keeps its base config. For svc-ai-litellm round 1 that meant web-app-keycloak ran with the base realm.totp_enabled true, so CONFIGURE_TOTP became a default required action and every SSO persona sat in a 300s predicate timeout until the job hit its wall clock at 20086s. payload.py states the intent this restores: the role pulling a dependency in is what decides how lean the dependency runs. variant_scope keeps the decision next to the closure it filters and excludes the three lab infrastructure roles, whose payload would replace group-membership expressions such as {{ 'svc-swarm-node' in group_names }} with an unconditional true in every round. Measured: round 1 goes from 1 to 29 payload apps and keycloak from absent to totp_enabled false, while the deployed closure stays 32. Repo-wide 62 of 280 rounds gain a real setting, dominated by postgres exposed in 48 and mariadb in 22; no round loses one, and no gained payload enables a service whose provider is not deployed. With disable set the scope shrinks instead, which is correct because a pruned provider is not deployed. Co-Authored-By: Claude Opus 5 (1M context) --- utils/tests/swarm/derive_includes.py | 15 +++++++++++++++ utils/tests/swarm/matrix.py | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/utils/tests/swarm/derive_includes.py b/utils/tests/swarm/derive_includes.py index 70cbf04f5f..c2b7cd5e2e 100755 --- a/utils/tests/swarm/derive_includes.py +++ b/utils/tests/swarm/derive_includes.py @@ -193,6 +193,21 @@ def derive_includes( return sorted(found) +def variant_scope(app_id: str, *, variants: dict[str, int] | None = None) -> list[str]: + """Deployed roles for APP_ID that must carry their own variant config. + + Args: + app_id: role whose closure is resolved. + variants: active ``{app_id: variant_index}`` map, as ``derive_includes``. + """ + explicit = set(_EXPLICIT_INCLUDES) - {app_id} + return [ + role + for role in derive_includes(app_id, variants=variants) + if role not in explicit + ] + + def _disabled_provider_roles(service_registry: dict[str, Any]) -> set[str]: """Provider roles the ``disable`` env removes from the closure, so the swarm include set matches what the provision-time services_disabler diff --git a/utils/tests/swarm/matrix.py b/utils/tests/swarm/matrix.py index 3edf062f6b..40ffe31bc8 100644 --- a/utils/tests/swarm/matrix.py +++ b/utils/tests/swarm/matrix.py @@ -29,7 +29,7 @@ import time from utils import PROJECT_ROOT from utils.env.runtime import mem_available_mb, mem_stall_pct, mem_total_mb from utils.storage.constrained import host_storage_constrained -from utils.tests.swarm.derive_includes import derive_includes +from utils.tests.swarm.derive_includes import derive_includes, variant_scope from utils.tests.swarm.write.extras import ensure_swarm_keypairs _SWARM_DIR = PROJECT_ROOT / "scripts" / "tests" / "deploy" / "swarm" @@ -397,7 +397,7 @@ def main(argv: list[str] | None = None) -> int: variant_payloads = _resolve_variant_payloads( roles_dir=_ROLES_DIR, - include=round_include, + include=variant_scope(app_id, variants=round_variants), active_variants=round_variants, ) from utils.tests.swarm.backup_repos import backup_provider_ips From 9164ab0c597531b7af13cab2cdb8355263478bd4 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 13:11:46 +0200 Subject: [PATCH 070/197] test(swarm-matrix): pin that a deployed app keeps its declared variant override Two guards over four primaries. svc-ai-litellm, web-app-bigbluebutton and web-app-peertube are the three whose round-1 include collapses to the primary alone, which is the only shape under which a deployed dependency can lose its payload; web-app-nextcloud is the ordinary control. One planner call costs 1.4s, so sweeping all 117 variant-carrying roles would cost 160s and buy nothing the three collapsing rounds do not already cover. The first attempt asserted round_include as a subset of the payload scope. That invariant only holds without disable: a clearnet row prunes the tor provider and everything reachable only through it, so the scope legitimately shrinks below the planner's include. The assertion now runs the other way, that every deployed app with a non-empty override receives it, which holds in both network modes. Both guards were watched failing before being kept: cutting web-app-keycloak and svc-net-tor out of variant_scope turns them red, and the suite passes with disable set as well as unset. Co-Authored-By: Claude Opus 5 (1M context) --- .../workflows/test_variant_payload_scope.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/integration/meta/workflows/test_variant_payload_scope.py diff --git a/tests/integration/meta/workflows/test_variant_payload_scope.py b/tests/integration/meta/workflows/test_variant_payload_scope.py new file mode 100644 index 0000000000..00a9d18967 --- /dev/null +++ b/tests/integration/meta/workflows/test_variant_payload_scope.py @@ -0,0 +1,96 @@ +"""The swarm matrix bakes a variant payload for every app it deploys.""" + +from __future__ import annotations + +import unittest +from typing import ClassVar + +from cli.administration.deploy.development.inventory.payload import ( + _resolve_variant_payloads, + get_variant_overrides_only, +) +from cli.administration.deploy.development.inventory.planner import ( + plan_dev_inventory_matrix, +) +from utils.cache.files import PROJECT_ROOT +from utils.tests.swarm.derive_includes import ( + _EXPLICIT_INCLUDES, + derive_includes, + variant_scope, +) + +_ROLES_DIR = str(PROJECT_ROOT / "roles") + + +def _rounds(primary: str) -> list[tuple]: + return plan_dev_inventory_matrix( + roles_dir=_ROLES_DIR, + primary_apps=[primary], + base_inventory_dir="/tmp/variant-payload-scope", + ) + + +class TestVariantPayloadScope(unittest.TestCase): + PRIMARIES: ClassVar[tuple[str, ...]] = ( + "svc-ai-litellm", + "web-app-bigbluebutton", + "web-app-peertube", + "web-app-nextcloud", + ) + + def test_every_deployed_app_receives_its_declared_override(self) -> None: + overrides = get_variant_overrides_only(roles_dir=_ROLES_DIR) + for primary in self.PRIMARIES: + for round_index, _inv, round_variants, _include, *_ in _rounds(primary): + scope = variant_scope(primary, variants=round_variants) + payloads = _resolve_variant_payloads( + roles_dir=_ROLES_DIR, + include=scope, + active_variants=round_variants, + ) + for app in derive_includes(primary, variants=round_variants): + declared = overrides.get(app) or [] + if not declared: + continue + index = round_variants.get(app, 0) + if not 0 <= index < len(declared): + index = 0 + if not declared[index] or app in _EXPLICIT_INCLUDES: + continue + self.assertIn( + app, + payloads, + f"{primary}#{round_index} deploys {app}, whose variants.yml " + f"declares an override, but the inventory bakes none, so it " + f"silently runs on its base config", + ) + + def test_a_dependency_keeps_its_override_when_the_include_collapses(self) -> None: + primary = "svc-ai-litellm" + for round_index, _inv, round_variants, round_include, *_ in _rounds(primary): + if "web-app-keycloak" not in variant_scope( + primary, variants=round_variants + ): + continue + payloads = _resolve_variant_payloads( + roles_dir=_ROLES_DIR, + include=variant_scope(primary, variants=round_variants), + active_variants=round_variants, + ) + realm = ( + payloads.get("web-app-keycloak", {}) + .get("services", {}) + .get("keycloak", {}) + .get("realm", {}) + ) + self.assertIs( + realm.get("totp_enabled"), + False, + f"{primary}#{round_index} deploys keycloak with " + f"{len(round_include)} planned app(s); without its variant override " + f"the realm demands CONFIGURE_TOTP and every SSO persona times out", + ) + + +if __name__ == "__main__": + unittest.main() From 05fcec52e754795b8da8f8c1acfa8faf79b90ab7 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 13:29:43 +0200 Subject: [PATCH 071/197] fix(planner): give every variant-carrying app the round's index, not only the include round_variants was built by walking round_include. When a primary's variant switches its couplings off the planner resolves no prerequisites, round_include collapses to the primary, and no dependency gets an index at all. derive_includes then evaluates them at base config, where an unrendered Jinja coupling counts as enabled, so the closure inflates to everything reachable instead of what the variant declares. Measured: svc-ai-litellm round 1 deploys 10 apps instead of 32, web-app-bigbluebutton round 1 and web-app-peertube round 1 drop from 34 to 10 and 11. Every other round of every role measured is unchanged, including all fourteen nextcloud rounds. The apps that leave are not dependencies of the round: svc-ai-litellm declares litellm, lmstudio, ollama and postgres and no sso at all, while bigbluebutton and peertube declare sso but switch it off in variant 1 together with dashboard, matomo, css, logout and prometheus. complexity/cli.py:300 already documents the rule this makes true: the planner hands each pulled-in dependency round_index if round_index is below its variant count else 0, and pulls_at_zero implements exactly that. The coverage model was describing behaviour the planner only delivered for apps inside the include. init.py filters the printed variant map to the apps actually shown, because round_variants now carries 266 entries and round 1 would otherwise print 112 of them on one line while ten apps deploy. Co-Authored-By: Claude Opus 5 (1M context) --- cli/administration/deploy/development/init.py | 6 +++--- cli/administration/deploy/development/inventory/planner.py | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cli/administration/deploy/development/init.py b/cli/administration/deploy/development/init.py index 6b69dd919b..de711ef911 100644 --- a/cli/administration/deploy/development/init.py +++ b/cli/administration/deploy/development/init.py @@ -175,9 +175,9 @@ def handler(args: argparse.Namespace) -> int: if len(plan) == 1: _, inv_dir, round_variants, include_R, _purge_set = plan[0] - non_zero = {a: i for a, i in round_variants.items() if i} - suffix = f" variants={non_zero}" if non_zero else "" shown = built_includes.get(inv_dir, include_R) + non_zero = {a: i for a, i in round_variants.items() if i and a in shown} + suffix = f" variants={non_zero}" if non_zero else "" print( f">>> Inventory initialized at {inv_dir} " f"(include={','.join(shown)} " @@ -189,8 +189,8 @@ def handler(args: argparse.Namespace) -> int: f"(primary_apps={','.join(primary_apps)}):" ) for round_index, inv_dir, round_variants, include_R, _purge_set in plan: - non_zero = {a: i for a, i in round_variants.items() if i} shown = built_includes.get(inv_dir, include_R) + non_zero = {a: i for a, i in round_variants.items() if i and a in shown} print( f" [round {round_index}] {inv_dir} " f"include={','.join(shown)} " diff --git a/cli/administration/deploy/development/inventory/planner.py b/cli/administration/deploy/development/inventory/planner.py index 8dcfdff18d..99e64a74c4 100644 --- a/cli/administration/deploy/development/inventory/planner.py +++ b/cli/administration/deploy/development/inventory/planner.py @@ -103,11 +103,10 @@ def plan_dev_inventory_matrix( round_include = per_variant_includes[round_index] round_variants = dict(primary_round_variants) - for dep in round_include: + for dep, dep_variants in variants_per_app.items(): if dep in round_variants: continue - dep_variants = variants_per_app.get(dep) or [{}] - dep_count = max(1, len(dep_variants)) + dep_count = max(1, len(dep_variants or [{}])) round_variants[dep] = round_index if round_index < dep_count else 0 inv_dir = f"{base}-{round_index}" if total_rounds > 1 else base From 8c60c5b6a68bfdc181605dcd7ba85013082defe4 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 14:03:14 +0200 Subject: [PATCH 072/197] style(swarm-matrix): mark the plan entry's include as unused e1d513a3c0 replaced include=round_include with variant_scope in the payload resolve and left the loop variable without a reader, which ruff B007 rejects: 'Loop control variable round_include not used within loop body'. grep confirms the unpacking at matrix.py:388 is now its only occurrence. Co-Authored-By: Claude Opus 5 (1M context) --- utils/tests/swarm/matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/tests/swarm/matrix.py b/utils/tests/swarm/matrix.py index 40ffe31bc8..4cd6fd4154 100644 --- a/utils/tests/swarm/matrix.py +++ b/utils/tests/swarm/matrix.py @@ -385,7 +385,7 @@ def main(argv: list[str] | None = None) -> int: round_index, inv_dir, round_variants, - round_include, + _round_include, round_purge_set, ) in enumerate(plan): inv_root = inv_dir.rstrip("/") From f247b29fd0f98f790a0dadfadac55a58924265ef Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 15:19:58 +0200 Subject: [PATCH 073/197] feat(hlth-csp): accept a declared status code instead of dropping the vhost The probe dropped every canonical and alias domain of a role whose server.status_codes.default named anything at or above 400, so a single by-design 4xx removed the whole vhost from CSP inspection. The checker now takes --accept-status =, which keeps the page navigated and its policy evaluated, so the declaration buys coverage instead of costing it. csp_accept_status reads the existing SPOT in utils/roles/applications/status_codes.py rather than restating the rule. That resolves per vhost, because server.status_codes is keyed like domains.canonical, so two domains of one role can differ; the old filter looked only at default and was blind to seaweedfs declaring its api, filer and master keys separately. The threshold is the checker's own: anything below 300 passes unflagged, so a declared redirect is named too. csp_skip_domains keeps only the half that has no vhost to probe at all, a canonical domain owned by a disabled service, and gains the tests that half never had. Measured: 14 domains move from skipped to probed, among them jenkins, mattermost, n8n, prometheus, mirror and xmpp. 28 domains carry an entry in total. Across the CI diagnostics of this run 473 of 476 observed responses were 200, the single infinito vhost answering 403 is api.seaweedfs, and it declares that code, so it stays green. Co-Authored-By: Claude Opus 5 (1M context) --- .../lookup_plugins/csp_accept_status.py | 83 ++++++++++++ .../lookup_plugins/csp_skip_domains.py | 49 +------ roles/sys-ctl-hlth-csp/meta/services.yml | 2 +- roles/sys-ctl-hlth-csp/tasks/00_core.yml | 4 + roles/sys-ctl-hlth-csp/vars/main.yml | 1 + .../lookup_plugins/test_csp_accept_status.py | 124 ++++++++++++++++++ .../lookup_plugins/test_csp_skip_domains.py | 93 ++++--------- 7 files changed, 243 insertions(+), 113 deletions(-) create mode 100644 roles/sys-ctl-hlth-csp/lookup_plugins/csp_accept_status.py create mode 100644 tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_accept_status.py diff --git a/roles/sys-ctl-hlth-csp/lookup_plugins/csp_accept_status.py b/roles/sys-ctl-hlth-csp/lookup_plugins/csp_accept_status.py new file mode 100644 index 0000000000..56a815d8ab --- /dev/null +++ b/roles/sys-ctl-hlth-csp/lookup_plugins/csp_accept_status.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from ansible.plugins.loader import lookup_loader +from ansible.plugins.lookup import LookupBase + +from utils.roles.applications.config import get +from utils.roles.applications.status_codes import declared_status_codes + +_CHECKER_ACCEPTS_BELOW = 300 + + +def _domains(value: Any) -> list[str]: + if isinstance(value, str): + return [value] if value else [] + if isinstance(value, Mapping): + out: list[str] = [] + for entry in value.values(): + out.extend(_domains(entry)) + return out + if isinstance(value, (list, tuple, set)): + out = [] + for entry in value: + out.extend(_domains(entry)) + return out + return [] + + +class LookupModule(LookupBase): + """Return ``=[,]`` for every declared non-2xx status. + + The checker treats anything outside 2xx as unreachable, so a vhost that + serves a 4xx by design needs its code named. Naming it keeps the page + under CSP inspection, which excluding the domain would not. + + The declaration is per vhost: ``server.status_codes`` is keyed like + ``domains.canonical``, so two domains of one role can differ. + """ + + def run( + self, + terms: list[Any] | None, + variables: dict[str, Any] | None = None, + **kwargs: Any, + ) -> list[list[str]]: + applications = lookup_loader.get( + "applications", + loader=self._loader, + templar=getattr(self, "_templar", None), + ).run( + [], + variables=variables + or getattr(self._templar, "available_variables", {}) + or {}, + )[0] + + if not isinstance(applications, Mapping): + return [[]] + + accepted: dict[str, set[int]] = {} + for app_id in applications: + domains = _domains( + get(applications, app_id, "domains.canonical", strict=False, default=[]) + ) + _domains( + get(applications, app_id, "domains.aliases", strict=False, default=[]) + ) + for domain in domains: + codes = { + code + for code in declared_status_codes(applications, app_id, domain) + if code >= _CHECKER_ACCEPTS_BELOW + } + if codes: + accepted.setdefault(domain, set()).update(codes) + + return [ + [ + f"{domain}={','.join(str(code) for code in sorted(codes))}" + for domain, codes in sorted(accepted.items()) + ] + ] diff --git a/roles/sys-ctl-hlth-csp/lookup_plugins/csp_skip_domains.py b/roles/sys-ctl-hlth-csp/lookup_plugins/csp_skip_domains.py index 04eee7c474..16af34bded 100644 --- a/roles/sys-ctl-hlth-csp/lookup_plugins/csp_skip_domains.py +++ b/roles/sys-ctl-hlth-csp/lookup_plugins/csp_skip_domains.py @@ -32,21 +32,6 @@ def _to_list(x: Any) -> list[str]: return [] -def _has_code_ge_400(codes: Any) -> bool: - if codes is None: - return False - if not isinstance(codes, (list, tuple, set)): - codes = [codes] - for c in codes: - try: - n = int(c) - except (TypeError, ValueError): - continue - if n >= 400: - return True - return False - - def _selection_from(group_names: Any) -> set[str]: if isinstance(group_names, (list, set, tuple)): return {str(x) for x in group_names if str(x)} @@ -58,11 +43,7 @@ def _selection_from(group_names: Any) -> set[str]: class LookupModule(LookupBase): """Return domains the CSP probe should skip. - Skips canonical + alias domains of every application whose - ``server.status_codes.default`` declares any HTTP code >= 400 - (e.g. federation-only roles that legitimately serve 4xx at ``/``). - - Additionally skips canonical domains owned by a disabled service: a + Skips canonical domains owned by a disabled service: a service entry may declare ``domains: [, ...]``; when its ``enabled`` resolves falsy those canonicals get no vhost (e.g. the seaweedfs filer/master frontend on an onion node), so probing them can @@ -121,32 +102,4 @@ class LookupModule(LookupBase): if not selection or app_id in selection: skip |= self._service_disabled_domains(applications, app_id) - default = get( - applications, - app_id, - "server.status_codes.default", - strict=False, - default=None, - ) - if not _has_code_ge_400(default): - continue - - canonical = get( - applications, - app_id, - "domains.canonical", - strict=False, - default=[], - ) - aliases = get( - applications, - app_id, - "domains.aliases", - strict=False, - default=[], - ) - for d in _to_list(canonical) + _to_list(aliases): - if d: - skip.add(d) - return [sorted(skip)] diff --git a/roles/sys-ctl-hlth-csp/meta/services.yml b/roles/sys-ctl-hlth-csp/meta/services.yml index acec2b10aa..bb5a8b194a 100644 --- a/roles/sys-ctl-hlth-csp/meta/services.yml +++ b/roles/sys-ctl-hlth-csp/meta/services.yml @@ -8,4 +8,4 @@ csp: categories: [web] csp-checker: image: ghcr.io/kevinveenbirkenbach/csp-checker - version: 2.3.0 + version: 3.0.0 diff --git a/roles/sys-ctl-hlth-csp/tasks/00_core.yml b/roles/sys-ctl-hlth-csp/tasks/00_core.yml index 4ca506c279..2d965b1a5c 100644 --- a/roles/sys-ctl-hlth-csp/tasks/00_core.yml +++ b/roles/sys-ctl-hlth-csp/tasks/00_core.yml @@ -60,6 +60,10 @@ '--skip-domain ' ~ (HEALTH_CSP_SKIP_DOMAINS | join(' ')), '' ), + (HEALTH_CSP_ACCEPT_STATUS | length > 0) | ternary( + '--accept-status ' ~ (HEALTH_CSP_ACCEPT_STATUS | join(' ')), + '' + ), (lookup('config', 'svc-net-tor', 'services.tor.node', '') | length > 0) | ternary( '--tor-proxy=socks5://' ~ lookup('tor_socks') ~ ' --onion-timeout=' ~ HEALTH_CSP_ONION_TIMEOUT_MS, diff --git a/roles/sys-ctl-hlth-csp/vars/main.yml b/roles/sys-ctl-hlth-csp/vars/main.yml index 61cad831c2..ea8d4a0ce0 100644 --- a/roles/sys-ctl-hlth-csp/vars/main.yml +++ b/roles/sys-ctl-hlth-csp/vars/main.yml @@ -3,6 +3,7 @@ system_service_id: sys-ctl-hlth-csp HEALTH_CSP_ONION_TIMEOUT_MS: "{{ (HEALTH_CSP_TIMEOUT_MS | int) * (lookup('config', 'svc-net-tor', 'services.tor.timeout_multipliers')[lookup('config', 'svc-net-tor', 'services.tor.flavor')] | int) }}" HEALTH_CSP_ONION_DOMAIN_SECONDS: "{{ ((HEALTH_CSP_ONION_TIMEOUT_MS | int) // 1000) + (HEALTH_CSP_DOMAIN_RESERVE_SECONDS | int) }}" +HEALTH_CSP_ACCEPT_STATUS: "{{ lookup('csp_accept_status') }}" HEALTH_CSP_SKIP_DOMAINS: >- {{ ( lookup('csp_skip_domains', group_names=lookup('deployment').deployed) diff --git a/tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_accept_status.py b/tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_accept_status.py new file mode 100644 index 0000000000..3378cc06e2 --- /dev/null +++ b/tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_accept_status.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import importlib.util +import sys +import unittest +import unittest.mock as mock +from unittest.mock import patch + +from . import PROJECT_ROOT + + +def _load_module(rel_path: str, name: str): + path = PROJECT_ROOT / rel_path + spec = importlib.util.spec_from_file_location(name, str(path)) + mod = importlib.util.module_from_spec(spec) + assert spec and spec.loader + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + +class _DummyTemplar: + def __init__(self, available_variables: dict | None = None): + self.available_variables = available_variables or {} + + +class CspAcceptStatusLookupTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.module = _load_module( + "roles/sys-ctl-hlth-csp/lookup_plugins/csp_accept_status.py", + "csp_accept_status_lookup", + ) + + def _run(self, applications, **kwargs): + lm = self.module.LookupModule() + lm._templar = _DummyTemplar() + lm._loader = mock.MagicMock() + with patch.object(self.module, "lookup_loader") as loader_mock: + loader_mock.get.return_value = mock.MagicMock( + run=lambda *_a, **_k: [applications] + ) + return lm.run(terms=[], variables={}, **kwargs)[0] + + def test_empty_applications_returns_empty_list(self): + self.assertEqual(self._run({}), []) + + def test_non_mapping_applications_returns_empty(self): + self.assertEqual(self._run([]), []) + + def test_a_declared_4xx_is_named_with_its_domain(self): + apps = { + "web-svc-mirror": { + "domains": {"canonical": ["mirror.example.com"]}, + "server": {"status_codes": {"default": [404]}}, + } + } + self.assertEqual(self._run(apps), ["mirror.example.com=404"]) + + def test_aliases_are_named_too(self): + apps = { + "web-app-bridgy": { + "domains": { + "canonical": ["bridgy.example.com"], + "aliases": ["fed.example.com"], + }, + "server": {"status_codes": {"default": [200, 404]}}, + } + } + self.assertEqual( + self._run(apps), + ["bridgy.example.com=404", "fed.example.com=404"], + ) + + def test_a_2xx_only_declaration_names_nothing(self): + apps = { + "web-app-foo": { + "domains": {"canonical": ["foo.example.com"]}, + "server": {"status_codes": {"default": [200]}}, + } + } + self.assertEqual(self._run(apps), []) + + def test_a_redirect_is_named_because_the_checker_rejects_it(self): + apps = { + "web-app-foo": { + "domains": {"canonical": ["foo.example.com"]}, + "server": {"status_codes": {"default": [200, 301]}}, + } + } + self.assertEqual(self._run(apps), ["foo.example.com=301"]) + + def test_two_vhosts_of_one_role_keep_their_own_codes(self): + apps = { + "web-app-split": { + "domains": { + "canonical": { + "api": "api.example.com", + "web": "web.example.com", + } + }, + "server": { + "status_codes": { + "api": [405], + "web": [200], + "default": [200], + } + }, + } + } + self.assertEqual(self._run(apps), ["api.example.com=405"]) + + def test_several_codes_of_one_domain_are_joined(self): + apps = { + "web-app-foo": { + "domains": {"canonical": ["foo.example.com"]}, + "server": {"status_codes": {"default": [200, 403, 404]}}, + } + } + self.assertEqual(self._run(apps), ["foo.example.com=403,404"]) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_skip_domains.py b/tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_skip_domains.py index 80defa526d..8fcc53d0c1 100644 --- a/tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_skip_domains.py +++ b/tests/unit/python/roles/sys-ctl-hlth-csp/lookup_plugins/test_csp_skip_domains.py @@ -24,6 +24,13 @@ class _DummyTemplar: self.available_variables = available_variables or {} +def _app(service_enabled, domain="filer.example.com"): + return { + "server": {"domains": {"canonical": {"filer": domain}}}, + "services": {"filer": {"enabled": service_enabled, "domains": ["filer"]}}, + } + + class CspSkipDomainsLookupTests(unittest.TestCase): @classmethod def setUpClass(cls): @@ -45,89 +52,47 @@ class CspSkipDomainsLookupTests(unittest.TestCase): def test_empty_applications_returns_empty_list(self): self.assertEqual(self._run({}), []) - def test_app_with_status_code_ge_400_skips_canonical_and_aliases(self): - apps = { - "web-app-bridgy": { - "domains": { - "canonical": ["bridgy.example.com"], - "aliases": ["fed.example.com"], - }, - "server": { - "status_codes": {"default": [200, 404]}, - }, - } - } - self.assertEqual(self._run(apps), ["bridgy.example.com", "fed.example.com"]) + def test_non_mapping_applications_returns_empty(self): + self.assertEqual(self._run([]), []) - def test_app_without_status_code_ge_400_is_not_skipped(self): + def test_a_disabled_service_hides_its_canonical_domain(self): + self.assertEqual( + self._run({"web-app-seaweedfs": _app(False)}), + ["filer.example.com"], + ) + + def test_an_enabled_service_keeps_its_domain_under_probe(self): + self.assertEqual(self._run({"web-app-seaweedfs": _app(True)}), []) + + def test_a_service_without_domains_is_ignored(self): apps = { "web-app-foo": { - "domains": {"canonical": ["foo.example.com"]}, - "server": { - "status_codes": {"default": [200]}, - }, + "server": {"domains": {"canonical": {"web": "foo.example.com"}}}, + "services": {"worker": {"enabled": False}}, } } self.assertEqual(self._run(apps), []) - def test_app_without_status_codes_field_is_not_skipped(self): - apps = {"web-app-foo": {"domains": {"canonical": ["foo.example.com"]}}} - self.assertEqual(self._run(apps), []) - - def test_a_declared_4xx_is_honoured_outside_the_selection(self): + def test_the_selection_filters_which_apps_are_inspected(self): apps = { - "web-app-bridgy": { - "domains": {"canonical": ["bridgy.example.com"]}, - "server": { - "status_codes": {"default": [404]}, - }, - }, - "web-svc-mirror": { - "domains": {"canonical": ["mirror.example.com"]}, - "server": { - "status_codes": {"default": [404]}, - }, - }, + "web-app-a": _app(False, "a.example.com"), + "web-app-b": _app(False, "b.example.com"), } self.assertEqual( - self._run(apps, group_names=["web-app-bridgy"]), - ["bridgy.example.com", "mirror.example.com"], + self._run(apps, group_names=["web-app-a"]), + ["a.example.com"], ) def test_group_names_csv_string_is_accepted(self): apps = { - "a": { - "domains": {"canonical": ["a.example.com"]}, - "server": { - "status_codes": {"default": [404]}, - }, - }, - "b": { - "domains": {"canonical": ["b.example.com"]}, - "server": { - "status_codes": {"default": [404]}, - }, - }, + "web-app-a": _app(False, "a.example.com"), + "web-app-b": _app(False, "b.example.com"), } self.assertEqual( - self._run(apps, group_names="a,b"), + self._run(apps, group_names="web-app-a,web-app-b"), ["a.example.com", "b.example.com"], ) - def test_non_4xx_codes_dont_count(self): - apps = { - "web-app-foo": { - "domains": {"canonical": ["foo.example.com"]}, - "server": { - "status_codes": {"default": [200, 301, 302]}, - }, - } - } - self.assertEqual(self._run(apps), []) - - def test_non_mapping_applications_returns_empty(self): - self.assertEqual(self._run([]), []) - if __name__ == "__main__": unittest.main(verbosity=2) From 095ea5e882a7725709265c044ad64fbed6ac9224 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 15:28:24 +0200 Subject: [PATCH 074/197] refactor(ai): preload one small chat model everywhere instead of a CI-only list The catalogue held three models and the development inventory carried a second, CI-only list that both AI roles pulled in through their variant 0. That made the model a variant concern and split one fact across a group_var, an inventory file and two meta files. AI_LOCAL_MODELS now holds the single entry the CI list held, qwen2.5 0.5b instruct at 531 MB in Q8_0, which lms get is already proven to fetch in this stack. AI_LOCAL_MODELS_CI and both variant overrides are gone, so variant 0 states only container_backup and the memory limits. The dropped entries have no consumer: llama3, mistral and nomic appear solely in test fixtures that stub the lookups, and nothing requests an embedding model, the only embed matches being the filters that exclude one when picking the chat model. LITELLM_CHAT_MODEL takes the first non-embed alias, so it resolves and LITELLM_USABLE stays true. Note this changes production, not only CI: a deployment no longer pulls a 7B model or an embedding model. It also shrinks the pre-pull that burned 18796s in job 104622514450 from gigabytes to 531 MB. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/16_ai.yml | 12 +++--------- inventories/development/default.yml | 5 ----- roles/svc-ai-lmstudio/meta/variants.yml | 1 - roles/svc-ai-ollama/meta/variants.yml | 1 - 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index 65c6f27bcd..5a5c5f2870 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -22,15 +22,9 @@ LITELLM_LMSTUDIO_BACKEND: >- and (lookup('config', 'svc-ai-litellm', 'services.lmstudio.shared') | bool) }} AI_LOCAL_MODELS: - - alias: llama3:latest - name: llama-3.2-3b-instruct - source: https://huggingface.co/lmstudio-community/Llama-3.2-3B-Instruct-GGUF - - alias: mistral:latest - name: mistral-7b-instruct-v0.3 - source: https://huggingface.co/lmstudio-community/Mistral-7B-Instruct-v0.3-GGUF - - alias: nomic-embed-text:latest - name: text-embedding-nomic-embed-text-v1.5 - source: https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF + - alias: qwen2.5:0.5b + name: qwen2.5-0.5b-instruct + source: https://huggingface.co/lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF AI_REMOTE_ALIASES: >- {{ ['openai/gpt-4o-mini'] * (lookup('api_enabled', 'openai') | int) diff --git a/inventories/development/default.yml b/inventories/development/default.yml index 09aa0a54dd..e498260df5 100644 --- a/inventories/development/default.yml +++ b/inventories/development/default.yml @@ -35,11 +35,6 @@ design: colors: base: "#FFA500" -AI_LOCAL_MODELS_CI: - - alias: qwen2.5:0.5b - name: qwen2.5-0.5b-instruct - source: https://huggingface.co/lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF - API: openai: api_key: "{{ lookup('env', 'INFINITO_API_OPENAI_KEY') }}" diff --git a/roles/svc-ai-lmstudio/meta/variants.yml b/roles/svc-ai-lmstudio/meta/variants.yml index 5ffa7b228c..5a12caf234 100644 --- a/roles/svc-ai-lmstudio/meta/variants.yml +++ b/roles/svc-ai-lmstudio/meta/variants.yml @@ -6,6 +6,5 @@ container_backup: enabled: true lmstudio: - preload_models: "{{ AI_LOCAL_MODELS_CI }}" mem_reservation: 1g mem_limit: 2g diff --git a/roles/svc-ai-ollama/meta/variants.yml b/roles/svc-ai-ollama/meta/variants.yml index 4e89b5aa75..6b9ad2d919 100644 --- a/roles/svc-ai-ollama/meta/variants.yml +++ b/roles/svc-ai-ollama/meta/variants.yml @@ -11,6 +11,5 @@ container_backup: enabled: true ollama: - preload_models: "{{ AI_LOCAL_MODELS_CI }}" mem_reservation: 1g mem_limit: 2g From 08b4d0f2fff2603c6072479f509b4ec46b64e5c0 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 17:31:16 +0200 Subject: [PATCH 075/197] fix(svc-net-tor): render the onion forward flags before reading them 2f533eb279 read TOR_ONION_SSH_ENABLED straight out of the lookup's variables, but group vars reach a lookup unrendered, so the flag arrived as its own template text and boolean() refused it. The torrc render aborted, and with it every tor row: 14 jobs across both runs on 095ea5e882 failed with the same line, Baserow#0 in compose and Nextcloud#1 in swarm among them. Quoted from job 104843161609: The lookup plugin 'tor_extra_ports' failed: The value '{{ lookup('config', application_id, 'services.tor.forwards.ssh') | bool }}' is not a valid boolean. Reproduced locally: boolean() on that literal raises with the identical message. tor_egress_clients.py:43 in this same role has resolved its own references through templar.template(trust_as_template(...)) since bf32f57b85. _flag now does the same and names the variable when the value is still not a boolean afterwards. The unit tests passed real booleans, which is why they stayed green through seven gate runs; the new case sends template text through a templar, the shape production delivers. No lint or unit test could have caught this, because it only appears once ansible hands the variable over unrendered. Co-Authored-By: Claude Opus 5 (1M context) --- .../lookup_plugins/tor_extra_ports.py | 22 ++++++++++++++- .../roles/svc-net-tor/test_extra_ports.py | 27 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/roles/svc-net-tor/lookup_plugins/tor_extra_ports.py b/roles/svc-net-tor/lookup_plugins/tor_extra_ports.py index db1b84bb45..f5f90a95d1 100644 --- a/roles/svc-net-tor/lookup_plugins/tor_extra_ports.py +++ b/roles/svc-net-tor/lookup_plugins/tor_extra_ports.py @@ -28,6 +28,7 @@ from ansible.errors import AnsibleError from ansible.module_utils.parsing.convert_bool import boolean from ansible.plugins.loader import lookup_loader from ansible.plugins.lookup import LookupBase +from ansible.template import trust_as_template FLAGGED_PORTS: tuple[tuple[str, int], ...] = ( ("TOR_ONION_SSH_ENABLED", 22), @@ -35,6 +36,25 @@ FLAGGED_PORTS: tuple[tuple[str, int], ...] = ( ) +def _flag(templar: Any, name: str, value: Any) -> bool: + """A declared flag as a boolean, with its Jinja resolved first. + + Args: + templar: the lookup's templar, or None when it has none. + name: the variable the flag came from, for the error message. + value: whatever the variable holds; group vars reach a lookup + unrendered, so a reference arrives as its own template text. + """ + if isinstance(value, str) and "{{" in value and templar is not None: + value = templar.template(trust_as_template(value)) + try: + return boolean(value) + except TypeError as exc: + raise AnsibleError( + f"lookup('tor_extra_ports'): {name} is not a boolean: {value!r}" + ) from exc + + def _entry(port: int) -> dict[str, Any]: """One forward, in the shape ``torrc.j2`` consumes. @@ -61,7 +81,7 @@ class LookupModule(LookupBase): for name, port in FLAGGED_PORTS: if name not in variables: raise AnsibleError(f"lookup('tor_extra_ports'): {name} is not defined") - if boolean(variables[name]): + if _flag(getattr(self, "_templar", None), name, variables[name]): entries.append(_entry(port)) derived = lookup_loader.get( diff --git a/tests/unit/python/roles/svc-net-tor/test_extra_ports.py b/tests/unit/python/roles/svc-net-tor/test_extra_ports.py index 2086e5ca12..e92fa52112 100644 --- a/tests/unit/python/roles/svc-net-tor/test_extra_ports.py +++ b/tests/unit/python/roles/svc-net-tor/test_extra_ports.py @@ -49,6 +49,16 @@ class _DerivedPorts: return [self.entries] +class _Templar: + """Resolves a group_var reference the way the play's templar does.""" + + def __init__(self, rendered: Any): + self.rendered = rendered + + def template(self, _value): + return self.rendered + + class TestExtraPorts(unittest.TestCase): @classmethod def setUpClass(cls) -> None: @@ -63,6 +73,23 @@ class TestExtraPorts(unittest.TestCase): ): return lookup.run([], variables={**ENABLED, **variables})[0] + def test_a_flag_that_arrives_as_template_text_is_still_read(self) -> None: + lookup = self.module.LookupModule() + lookup._templar = _Templar(True) + lookup._loader = None + variables = dict.fromkeys(ENABLED, "{{ lookup('config', application_id, 'services.tor.forwards.x') | bool }}") + with mock.patch.object( + self.module.lookup_loader, "get", return_value=_DerivedPorts([]) + ): + ports = lookup.run([], variables=variables)[0] + self.assertIn( + HTTP_ENTRY, + ports, + "group vars reach a lookup unrendered, so the flag arrives as its own " + "template text; reading it without templating aborts the torrc render " + "on every tor row", + ) + def test_http_is_forwarded_without_the_proxy_in_the_inventory(self) -> None: self.assertIn( HTTP_ENTRY, From e177a7c4a95054bf8b0a5302a0b3aa21efad41a8 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 17:37:34 +0200 Subject: [PATCH 076/197] style(svc-net-tor): wrap the flag fixture the way autoformat does 08b4d0f2ff was committed while the gate was still running, so ruff format wrapped the dict.fromkeys call after the commit had already been written. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/python/roles/svc-net-tor/test_extra_ports.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/python/roles/svc-net-tor/test_extra_ports.py b/tests/unit/python/roles/svc-net-tor/test_extra_ports.py index e92fa52112..4deb747de1 100644 --- a/tests/unit/python/roles/svc-net-tor/test_extra_ports.py +++ b/tests/unit/python/roles/svc-net-tor/test_extra_ports.py @@ -77,7 +77,10 @@ class TestExtraPorts(unittest.TestCase): lookup = self.module.LookupModule() lookup._templar = _Templar(True) lookup._loader = None - variables = dict.fromkeys(ENABLED, "{{ lookup('config', application_id, 'services.tor.forwards.x') | bool }}") + variables = dict.fromkeys( + ENABLED, + "{{ lookup('config', application_id, 'services.tor.forwards.x') | bool }}", + ) with mock.patch.object( self.module.lookup_loader, "get", return_value=_DerivedPorts([]) ): From 60d187410a684ca0bfc9a2b57f87be676c8fc77d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 20:35:35 +0200 Subject: [PATCH 077/197] fix(hlth-csp): accept the status declarations the unit already passes The systemd unit has passed --accept-status since f247b29fd0, but the wrapper script never learned the flag. argparse rejected it, so the service failed with status=2/INVALIDARGUMENT on every host that deploys the role, in compose and in swarm alike. The checker consumes every token after a list flag that does not start with "--", and a URL does not, so both lists now share a single separator placed before the URLs. The previous placement sat directly behind --ignore-network-blocks-from and would have fed a second list straight into the URL parser. Co-Authored-By: Claude Opus 5 (1M context) --- roles/sys-ctl-hlth-csp/files/python/script.py | 21 ++++++ .../test_list_flags_are_separated.py | 55 ++++++++++++++ .../test_unit_flags_are_parsable.py | 75 +++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 tests/unit/python/roles/sys-ctl-hlth-csp/test_list_flags_are_separated.py create mode 100644 tests/unit/python/roles/sys-ctl-hlth-csp/test_unit_flags_are_parsable.py diff --git a/roles/sys-ctl-hlth-csp/files/python/script.py b/roles/sys-ctl-hlth-csp/files/python/script.py index c79e8ddb2b..761ff3159a 100755 --- a/roles/sys-ctl-hlth-csp/files/python/script.py +++ b/roles/sys-ctl-hlth-csp/files/python/script.py @@ -131,6 +131,7 @@ def build_docker_cmd( use_host_network: bool = True, proxy: str = "", timeout_ms: int = 0, + accept_status: list[str] | None = None, ) -> list[str]: cmd = ["container", "run", "--rm"] @@ -155,6 +156,12 @@ def build_docker_cmd( if ignore_network_blocks_from: cmd.append("--ignore-network-blocks-from") cmd.extend(ignore_network_blocks_from) + + if accept_status: + cmd.append("--accept-status") + cmd.extend(accept_status) + + if ignore_network_blocks_from or accept_status: cmd.append("--") cmd.extend(urls) @@ -170,6 +177,7 @@ def run_checker( use_host_network: bool = True, proxy: str = "", timeout_ms: int = 0, + accept_status: list[str] | None = None, ) -> int: """ Runs the CSP checker container and returns its exit code. @@ -186,6 +194,7 @@ def run_checker( use_host_network=use_host_network, proxy=proxy, timeout_ms=timeout_ms, + accept_status=accept_status, ) try: @@ -245,6 +254,17 @@ def main() -> None: "otherwise treat as unreachable." ), ) + parser.add_argument( + "--accept-status", + nargs="*", + default=[], + help=( + "Per-vhost status codes the probe must treat as healthy, as " + "=[,]. Declared in the role's " + "server.status_codes; everything below 300 passes without being " + "listed." + ), + ) parser.add_argument( "--onion-timeout", type=int, @@ -322,6 +342,7 @@ def main() -> None: use_host_network=not bool(args.no_host_network), proxy=batch_proxy, timeout_ms=batch_timeout, + accept_status=list(args.accept_status or []), ) rc = rc or batch_rc sys.exit(rc) diff --git a/tests/unit/python/roles/sys-ctl-hlth-csp/test_list_flags_are_separated.py b/tests/unit/python/roles/sys-ctl-hlth-csp/test_list_flags_are_separated.py new file mode 100644 index 0000000000..59fb813bef --- /dev/null +++ b/tests/unit/python/roles/sys-ctl-hlth-csp/test_list_flags_are_separated.py @@ -0,0 +1,55 @@ +import importlib.util +import unittest + +from . import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "roles" / "sys-ctl-hlth-csp" / "files" / "python" / "script.py" +URLS = ["https://a.test/", "https://b.test/"] + + +def _load(): + spec = importlib.util.spec_from_file_location("csp_wrapper", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _cmd(**kwargs): + return _load().build_docker_cmd( + image="img", + urls=URLS, + short_mode=False, + ignore_network_blocks_from=kwargs.get("ignore", []), + accept_status=kwargs.get("accept", []), + ) + + +class TestListFlagsAreSeparatedFromUrls(unittest.TestCase): + def test_accept_status_is_separated_from_the_urls(self): + cmd = _cmd(accept=["a.test=403"]) + + self.assertEqual( + cmd[cmd.index("--accept-status") + 2], + "--", + "the checker consumes every token after a list flag that does not " + "start with '--', and a URL does not, so the URLs would be read as " + "more status declarations", + ) + + def test_two_list_flags_share_a_single_separator(self): + cmd = _cmd(ignore=["x.test"], accept=["y.test=403"]) + + self.assertEqual(cmd.count("--"), 1) + self.assertLess(cmd.index("--"), cmd.index(URLS[0])) + + def test_without_a_list_flag_no_separator_is_emitted(self): + self.assertNotIn("--", _cmd()) + + def test_every_url_survives_the_separator(self): + cmd = _cmd(accept=["y.test=403"]) + + self.assertEqual(cmd[cmd.index("--") + 1 :], URLS) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/roles/sys-ctl-hlth-csp/test_unit_flags_are_parsable.py b/tests/unit/python/roles/sys-ctl-hlth-csp/test_unit_flags_are_parsable.py new file mode 100644 index 0000000000..759832c95a --- /dev/null +++ b/tests/unit/python/roles/sys-ctl-hlth-csp/test_unit_flags_are_parsable.py @@ -0,0 +1,75 @@ +import re +import subprocess +import sys +import unittest + +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +ROLE = PROJECT_ROOT / "roles" / "sys-ctl-hlth-csp" +TASKS = ROLE / "tasks" / "00_core.yml" +SCRIPT = ROLE / "files" / "python" / "script.py" + +_FLAG = re.compile(r"'\s*(--[a-z][a-z0-9-]*)") +_VALUE = { + "--nginx-config-dir": "/nonexistent", + "--image": "img", + "--tor-proxy": "socks5://127.0.0.1:9050", + "--onion-timeout": "1000", +} +_LIST_VALUE = { + "--ignore-network-blocks-from": "a.test", + "--skip-domain": "a.test", + "--accept-status": "a.test=403", +} + + +def _flags_the_unit_passes() -> set[str]: + return set(_FLAG.findall(read_text(str(TASKS)))) + + +def _invoke(flag: str) -> subprocess.CompletedProcess: + argv = [ + sys.executable, + str(SCRIPT), + "--nginx-config-dir", + "/nonexistent", + "--image", + "img", + ] + if flag in _LIST_VALUE: + argv.extend([flag, _LIST_VALUE[flag]]) + elif flag in _VALUE and flag not in ("--nginx-config-dir", "--image"): + argv.extend([flag, _VALUE[flag]]) + elif flag not in _VALUE: + argv.append(flag) + return subprocess.run(argv, capture_output=True, text=True, timeout=60, check=False) + + +class TestUnitFlagsAreParsable(unittest.TestCase): + def test_the_unit_passes_at_least_the_known_flags(self): + found = _flags_the_unit_passes() + + self.assertIn("--accept-status", found) + self.assertIn("--onion-timeout", found) + + def test_every_flag_the_unit_passes_is_accepted_by_the_script(self): + for flag in sorted(_flags_the_unit_passes()): + with self.subTest(flag=flag): + completed = _invoke(flag) + self.assertNotIn( + "unrecognized arguments", + completed.stderr, + f"the systemd unit passes {flag} but script.py rejects it, " + f"which fails the service with status=2/INVALIDARGUMENT", + ) + + def test_an_undeclared_flag_is_still_rejected(self): + completed = _invoke("--definitely-not-a-flag") + + self.assertIn("unrecognized arguments", completed.stderr) + + +if __name__ == "__main__": + unittest.main() From 26e54b8f8e147372b8466653bdc6ce235591c201 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 20:38:50 +0200 Subject: [PATCH 078/197] feat(install): pin the collections and stop refetching what is already there The requirements named seven collections without versions, so every install re-resolved them against galaxy.ansible.com and --force-with-deps redownloaded all 37 MB even when nothing had changed. Exact pins make the installed set checkable, and the check short-circuits the whole roundtrip: make install-ansible drops from 8m24s to 0.077s, and a Galaxy outage stops mattering once the pinned set is present. The git fallback is pinned to the same versions so it installs what the Galaxy file declares rather than whatever the branches currently hold. kewlfft.aur pins a commit because its tags carry no galaxy.yml and master has moved past the released version. The lint installer kept a second, weaker truth about the same directory, guessing from directory presence for three hardcoded collections; it now shares the version-exact check. Co-Authored-By: Claude Opus 5 (1M context) --- default.env | 3 + requirements/requirements.galaxy.yml | 17 ++-- requirements/requirements.git.yml | 14 ++-- scripts/install/ansible.sh | 62 ++++++++++---- .../python/utils/install/test_collections.py | 83 +++++++++++++++++++ utils/install/collections.py | 73 ++++++++++++++++ utils/install/lint/ansible/collections.py | 25 ++---- 7 files changed, 229 insertions(+), 48 deletions(-) create mode 100644 tests/unit/python/utils/install/test_collections.py create mode 100644 utils/install/collections.py diff --git a/default.env b/default.env index 440e46ac39..e98136c83e 100644 --- a/default.env +++ b/default.env @@ -16,6 +16,9 @@ # Git repository holding the operator's terminal aliases (an `aliases` file at repo root); developers override this with their own repo. INFINITO_ALIAS_REPOSITORY=https://github.com/kevinveenbirkenbach/alias +# Requirements file scripts/install/ansible.sh tries first, `galaxy` or `git`; the other one stays the fallback. +INFINITO_ANSIBLE_COLLECTIONS_SOURCE=galaxy + # Anthropic API key the development inventory feeds into API.anthropic.api_key; empty leaves the anthropic routes unpublished. INFINITO_API_ANTHROPIC_KEY="" diff --git a/requirements/requirements.galaxy.yml b/requirements/requirements.galaxy.yml index 33ae9d1493..99944bdf14 100644 --- a/requirements/requirements.galaxy.yml +++ b/requirements/requirements.galaxy.yml @@ -1,9 +1,16 @@ --- collections: - name: ansible.posix - - name: kewlfft.aur - - name: community.general - - name: community.docker - - name: hetzner.hcloud - - name: ansible.posix + version: 2.2.2 - name: ansible.mysql + version: 5.2.0 + - name: kewlfft.aur + version: 0.13.0 + - name: community.general + version: 13.4.0 + - name: community.docker + version: 5.3.0 + - name: community.library_inventory_filtering_v1 + version: 1.1.5 + - name: hetzner.hcloud + version: 7.0.1 diff --git a/requirements/requirements.git.yml b/requirements/requirements.git.yml index 31ddb7279d..5e6b9eaf0e 100644 --- a/requirements/requirements.git.yml +++ b/requirements/requirements.git.yml @@ -3,34 +3,34 @@ collections: - name: ansible.posix source: https://github.com/ansible-collections/ansible.posix.git type: git - version: main + version: 2.2.2 - name: ansible.mysql source: https://github.com/ansible-collections/ansible.mysql.git type: git - version: main + version: 5.2.0 - name: kewlfft.aur source: https://github.com/kewlfft/ansible-aur.git type: git - version: master + version: 7affd00193b03bf820ebcd8b13dfa81aab59803e - name: community.general source: https://github.com/ansible-collections/community.general.git type: git - version: main + version: 13.4.0 - name: community.docker source: https://github.com/ansible-collections/community.docker.git type: git - version: main + version: 5.3.0 - name: community.library_inventory_filtering_v1 source: https://github.com/ansible-collections/community.library_inventory_filtering.git type: git - version: stable-1 + version: 1.1.5 - name: hetzner.hcloud source: https://github.com/ansible-collections/hetzner.hcloud.git type: git - version: main + version: 7.0.1 diff --git a/scripts/install/ansible.sh b/scripts/install/ansible.sh index 4b347d570b..3f0152a001 100755 --- a/scripts/install/ansible.sh +++ b/scripts/install/ansible.sh @@ -16,30 +16,56 @@ INSTALL_TIMEOUT=15m GALAXY_REQ="requirements/requirements.galaxy.yml" GIT_REQ="requirements/requirements.git.yml" -while true; do +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +cd "${REPO_ROOT}" + +if [[ -z "${INFINITO_ANSIBLE_COLLECTIONS_SOURCE:-}" ]]; then + # shellcheck source=/dev/null + source <(grep -E '^INFINITO_ANSIBLE_COLLECTIONS_SOURCE=' default.env) +fi + +case "${INFINITO_ANSIBLE_COLLECTIONS_SOURCE:-galaxy}" in +galaxy) SOURCE_ORDER=("Galaxy:${GALAXY_REQ}" "Git:${GIT_REQ}") ;; +git) SOURCE_ORDER=("Git:${GIT_REQ}" "Galaxy:${GALAXY_REQ}") ;; +*) + echo "โŒ INFINITO_ANSIBLE_COLLECTIONS_SOURCE must be 'galaxy' or 'git'," \ + "got '${INFINITO_ANSIBLE_COLLECTIONS_SOURCE}'" >&2 + exit 2 + ;; +esac + +if MISSING="$("${PYTHON}" -m utils.install.collections "${GALAXY_REQ}" "${ANSIBLE_COLLECTIONS_DIR}")"; then + echo "โœ… Every pinned collection is already installed" + echo "๐ŸŽ‰ All collections are ready" + exit 0 +fi +echo "โ†’ Not satisfied yet: ${MISSING}" + +installed=0 +while ((installed == 0)); do echo "โ–ถ๏ธ Attempt ${ATTEMPT}/${MAX_ATTEMPTS}" - echo "๐ŸŒ Trying Galaxy source (${GALAXY_REQ})โ€ฆ" - if timeout --foreground "${INSTALL_TIMEOUT}" \ - "${PYTHON}" -m ansible.cli.galaxy collection install \ - -r "${GALAXY_REQ}" \ - -p "${ANSIBLE_COLLECTIONS_DIR}" \ - --force-with-deps; then + for entry in "${SOURCE_ORDER[@]}"; do + label="${entry%%:*}" + req="${entry#*:}" - echo "โœ… Collections installed successfully via Galaxy on attempt ${ATTEMPT}" - break - fi + echo "๐ŸŒ Trying ${label} source (${req})โ€ฆ" + if timeout --foreground "${INSTALL_TIMEOUT}" \ + "${PYTHON}" -m ansible.cli.galaxy collection install \ + -r "${req}" \ + -p "${ANSIBLE_COLLECTIONS_DIR}" \ + --force-with-deps; then - echo "โš ๏ธ Galaxy install failed on attempt ${ATTEMPT}" + echo "โœ… Collections installed successfully via ${label} on attempt ${ATTEMPT}" + installed=1 + break + fi - echo "๐Ÿ” Falling back to Git source (${GIT_REQ})โ€ฆ" - if timeout --foreground "${INSTALL_TIMEOUT}" \ - "${PYTHON}" -m ansible.cli.galaxy collection install \ - -r "${GIT_REQ}" \ - -p "${ANSIBLE_COLLECTIONS_DIR}" \ - --force-with-deps; then + echo "โš ๏ธ ${label} install failed on attempt ${ATTEMPT}" + done - echo "โœ… Collections installed successfully via Git fallback on attempt ${ATTEMPT}" + if ((installed == 1)); then break fi diff --git a/tests/unit/python/utils/install/test_collections.py b/tests/unit/python/utils/install/test_collections.py new file mode 100644 index 0000000000..ed581be0b7 --- /dev/null +++ b/tests/unit/python/utils/install/test_collections.py @@ -0,0 +1,83 @@ +import json +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from utils.install.collections import unsatisfied + +REQUIREMENTS = """--- +collections: + - name: community.general + version: 13.4.0 + - name: hetzner.hcloud + version: 7.0.1 +""" + + +def _write_requirements(root: Path, body: str = REQUIREMENTS) -> Path: + path = root / "requirements.galaxy.yml" + path.write_text(body, encoding="utf-8") + return path + + +def _install(collections_dir: Path, fqcn: str, version: str) -> None: + namespace, _, name = fqcn.partition(".") + target = collections_dir / "ansible_collections" / namespace / name + target.mkdir(parents=True) + (target / "MANIFEST.json").write_text( + json.dumps({"collection_info": {"version": version}}), encoding="utf-8" + ) + + +class TestUnsatisfied(unittest.TestCase): + def test_every_pin_installed_at_its_version_is_satisfied(self): + with TemporaryDirectory() as tmp: + root = Path(tmp) + requirements = _write_requirements(root) + _install(root, "community.general", "13.4.0") + _install(root, "hetzner.hcloud", "7.0.1") + + self.assertEqual(unsatisfied(requirements, root), []) + + def test_absent_collection_is_reported(self): + with TemporaryDirectory() as tmp: + root = Path(tmp) + requirements = _write_requirements(root) + _install(root, "community.general", "13.4.0") + + self.assertEqual(unsatisfied(requirements, root), ["hetzner.hcloud:7.0.1"]) + + def test_wrong_version_is_reported_although_the_directory_exists(self): + with TemporaryDirectory() as tmp: + root = Path(tmp) + requirements = _write_requirements(root) + _install(root, "community.general", "13.4.0") + _install(root, "hetzner.hcloud", "6.9.0") + + self.assertEqual(unsatisfied(requirements, root), ["hetzner.hcloud:7.0.1"]) + + def test_unpinned_entry_can_never_be_confirmed(self): + body = "---\ncollections:\n - name: community.general\n" + with TemporaryDirectory() as tmp: + root = Path(tmp) + requirements = _write_requirements(root, body) + _install(root, "community.general", "13.4.0") + + self.assertEqual(unsatisfied(requirements, root), ["community.general"]) + + def test_unreadable_manifest_is_reported(self): + with TemporaryDirectory() as tmp: + root = Path(tmp) + requirements = _write_requirements(root) + _install(root, "community.general", "13.4.0") + _install(root, "hetzner.hcloud", "7.0.1") + manifest = ( + root / "ansible_collections" / "hetzner" / "hcloud" / "MANIFEST.json" + ) + manifest.write_text("{not json", encoding="utf-8") + + self.assertEqual(unsatisfied(requirements, root), ["hetzner.hcloud:7.0.1"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/install/collections.py b/utils/install/collections.py new file mode 100644 index 0000000000..338072cfd4 --- /dev/null +++ b/utils/install/collections.py @@ -0,0 +1,73 @@ +"""Compare the pinned Galaxy requirements against what is on disk.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import yaml + + +def _installed_version(collections_dir: Path, namespace: str, name: str) -> str | None: + """The version recorded in a collection's manifest, or None when absent. + + Args: + collections_dir: the ``-p`` target ansible-galaxy installs into. + namespace: collection namespace, the part before the dot. + name: collection name, the part after the dot. + """ + manifest = ( + collections_dir / "ansible_collections" / namespace / name / "MANIFEST.json" + ) + try: + payload = json.loads(manifest.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + version = payload.get("collection_info", {}).get("version") + return str(version) if version else None + + +def unsatisfied(requirements_file: Path, collections_dir: Path) -> list[str]: + """Requirements that COLLECTIONS_DIR does not already hold at the pinned version. + + An entry without a pinned version can never be confirmed from disk and is + always reported, so the caller falls through to a real install. + + Args: + requirements_file: the requirements.galaxy.yml to read. + collections_dir: the ``-p`` target ansible-galaxy installs into. + """ + declared = yaml.safe_load(requirements_file.read_text(encoding="utf-8")) or {} + missing: list[str] = [] + for entry in declared.get("collections") or []: + if not isinstance(entry, dict): + missing.append(str(entry)) + continue + fqcn = str(entry.get("name", "")) + pinned = entry.get("version") + if not fqcn or "." not in fqcn or not pinned: + missing.append(fqcn or repr(entry)) + continue + namespace, _, name = fqcn.partition(".") + if _installed_version(collections_dir, namespace, name) != str(pinned): + missing.append(f"{fqcn}:{pinned}") + return missing + + +def main() -> int: + """Exit 0 when every pinned collection is present at its pinned version. + + Args: + argv[1]: path to requirements.galaxy.yml. + argv[2]: path to the collections directory. + """ + missing = unsatisfied(Path(sys.argv[1]), Path(sys.argv[2])) + if missing: + print(" ".join(missing)) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/install/lint/ansible/collections.py b/utils/install/lint/ansible/collections.py index 1edd2f83e2..17a2dfec9a 100644 --- a/utils/install/lint/ansible/collections.py +++ b/utils/install/lint/ansible/collections.py @@ -8,16 +8,13 @@ import time from pathlib import Path from utils.cache import PROJECT_ROOT +from utils.install.collections import unsatisfied from utils.install.primitives import log, warn _MAX_ATTEMPTS = 5 -def _collection_present(base_dir: Path, namespace: str, name: str) -> bool: - return (base_dir / "ansible_collections" / namespace / name).is_dir() - - -def _galaxy_install(requirements_file: str, base_dir: Path) -> bool: +def _galaxy_install(requirements_file: Path, base_dir: Path) -> bool: try: subprocess.run( [ @@ -25,7 +22,7 @@ def _galaxy_install(requirements_file: str, base_dir: Path) -> bool: "collection", "install", "-r", - requirements_file, + str(requirements_file), "-p", str(base_dir), "--force-with-deps", @@ -39,22 +36,14 @@ def _galaxy_install(requirements_file: str, base_dir: Path) -> bool: def ensure() -> None: collections_base_dir = Path("~/.ansible/collections").expanduser() - missing: list[str] = [] - - if not _collection_present(collections_base_dir, "community", "general"): - missing.append("community.general") - if not _collection_present(collections_base_dir, "hetzner", "hcloud"): - missing.append("hetzner.hcloud") - if not _collection_present(collections_base_dir, "kewlfft", "aur"): - missing.append("kewlfft.aur") + repo_root = Path(PROJECT_ROOT) + req_galaxy = repo_root / "requirements" / "requirements.galaxy.yml" + req_git = repo_root / "requirements" / "requirements.git.yml" + missing = unsatisfied(req_galaxy, collections_base_dir) if not missing: return - repo_root = Path(PROJECT_ROOT) - req_galaxy = str(repo_root / "requirements" / "requirements.galaxy.yml") - req_git = str(repo_root / "requirements" / "requirements.git.yml") - attempt = 1 while True: log( From ef45d69dc36e30336079ae9641411debd7bbcece Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 20:54:49 +0200 Subject: [PATCH 079/197] perf(image): build the collections into a layer the source cannot invalidate COPY . preceded every expensive RUN, so each commit rebuilt the whole chain and ansible-galaxy ran on every build of every distro. The collection install now sits in its own layer fed only by default.env, scripts/install, the dev-python shell scripts, requirements/ and the three files the version check imports. Everything that needs the full tree, the project's own distro package included, stays behind COPY .. Measured on a role-only change: the collection layer stays CACHED and the build drops to 3m10s against a CI baseline of 855-899s. An unchanged rebuild is 15 cached layers in 1.5s. The gha cache imported and exported nothing in a plain run step, so the export moves to the registry the job already authenticates against. ignore-error keeps a cache write failure from taking the build with it, which it otherwise does: proven against an unreachable registry, where the build fails hard without it and exits 0 with it. Co-Authored-By: Claude Opus 5 (1M context) --- Dockerfile | 18 ++++++++++++++++++ scripts/image/push.sh | 6 ++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b1f8bfc0f8..25e3f98710 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,24 @@ RUN set -euo pipefail; \ echo 'accept-flake-config = true' >> /etc/nix/nix.conf; \ fi +COPY default.env ${INFINITO_SRC_DIR}/default.env +COPY scripts/install ${INFINITO_SRC_DIR}/scripts/install +COPY roles/dev-python/files/shell ${INFINITO_SRC_DIR}/roles/dev-python/files/shell +COPY requirements ${INFINITO_SRC_DIR}/requirements +COPY utils/__init__.py ${INFINITO_SRC_DIR}/utils/__init__.py +COPY utils/install/__init__.py ${INFINITO_SRC_DIR}/utils/install/__init__.py +COPY utils/install/collections.py ${INFINITO_SRC_DIR}/utils/install/collections.py + +# hadolint ignore=DL3008,DL3033,DL3041,SC1090,SC3040 +RUN set -euo pipefail; \ + source <(grep -hE '^INFINITO_PYTHON_INSTALL_SCRIPT=' "${INFINITO_SRC_DIR}/default.env"); \ + /bin/bash "${INFINITO_SRC_DIR}/${INFINITO_PYTHON_INSTALL_SCRIPT:?}" ensure; \ + VENV="${INFINITO_VENV_DIR}" bash "${INFINITO_SRC_DIR}/scripts/install/venv.sh"; \ + ${PIP} install --upgrade pip setuptools wheel; \ + ${PIP} install ansible PyYAML; \ + ANSIBLE_COLLECTIONS_DIR="${HOME}/.ansible/collections" \ + bash "${INFINITO_SRC_DIR}/scripts/install/ansible.sh" + COPY . ${INFINITO_SRC_DIR} # hadolint ignore=DL3008,DL3033,DL3041,SC1090 diff --git a/scripts/image/push.sh b/scripts/image/push.sh index 4cb3ec606c..d6c2f90cc0 100755 --- a/scripts/image/push.sh +++ b/scripts/image/push.sh @@ -19,6 +19,8 @@ source "${script_dir}/../meta/env/load.sh" ghcr_owner="$(scripts/meta/resolve/repository/owner.sh)" repo_name="$("${script_dir}/../meta/resolve/repository/name.sh")" +cache_ref="ghcr.io/${ghcr_owner}/${repo_name}/${MATRIX_DISTRO}:buildcache" + max_attempts="${MAX_ATTEMPTS:-7}" retry_delay_seconds="${RETRY_DELAY_SECONDS:-20}" attempt=1 @@ -46,8 +48,8 @@ while true; do --build-arg "INFINITO_PYTHON_INSTALL_SCRIPT=${INFINITO_PYTHON_INSTALL_SCRIPT:?source scripts/meta/env/load.sh}" \ --build-arg "INFINITO_DOCKER_CLI_INSTALL_SCRIPT=${INFINITO_DOCKER_CLI_INSTALL_SCRIPT:?source scripts/meta/env/load.sh}" \ "${nix_arg[@]}" \ - --cache-from "type=gha,scope=${repo_name}-${MATRIX_DISTRO}" \ - --cache-to "type=gha,mode=max,scope=${repo_name}-${MATRIX_DISTRO}" \ + --cache-from "type=registry,ref=${cache_ref}" \ + --cache-to "type=registry,mode=max,ignore-error=true,ref=${cache_ref}" \ "${BUILD_CONTEXT_DIR}"; then echo "Build & push succeeded on attempt ${attempt}/${max_attempts}." break From e992175fccd8c83131267444d09b1e70462c95a3 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 20:58:35 +0200 Subject: [PATCH 080/197] feat(update): keep the pinned Ansible collections from ageing unnoticed Pinning the collections froze them, and nothing thaws a pin: dependabot reads pyproject.toml, the docker updater reads image tags, and neither sees a version inside requirements/. The new job resolves upstream releases with git ls-remote --tags and opens a PR the same way the pip, image and repository-ref updaters do. Both requirement files are bumped together, because the git fallback has to install what the Galaxy file declares; moving only one would break that. A collection whose two files disagree, or whose version is not a semver, is left alone -- that covers the branch ref and the commit pin, which state an intent a bumper must not guess. The job calls pr.sh rather than role_prs.sh: the latter splits per sub-directory and would open no PR at all for flat paths like requirements/*.yml. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/cron-update.yml | 47 ++++++ cli/contributing/update/galaxy/__init__.py | 5 + cli/contributing/update/galaxy/__main__.py | 58 ++++++++ scripts/github/update/galaxy_pins.sh | 13 ++ tests/unit/python/utils/update/test_galaxy.py | 134 +++++++++++++++++ utils/update/galaxy.py | 135 ++++++++++++++++++ 6 files changed, 392 insertions(+) create mode 100644 cli/contributing/update/galaxy/__init__.py create mode 100755 cli/contributing/update/galaxy/__main__.py create mode 100755 scripts/github/update/galaxy_pins.sh create mode 100644 tests/unit/python/utils/update/test_galaxy.py create mode 100644 utils/update/galaxy.py diff --git a/.github/workflows/cron-update.yml b/.github/workflows/cron-update.yml index cf43307701..6e827796f2 100644 --- a/.github/workflows/cron-update.yml +++ b/.github/workflows/cron-update.yml @@ -112,6 +112,53 @@ jobs: UPDATE_BASE_BRANCH: main run: bash scripts/github/update/open/role_prs.sh roles + update-galaxy-pins: + name: ๐ŸŒŒ Update pinned Ansible collections + if: vars.CI_ENABLE_AUTO_UPDATES == 'true' + runs-on: ubuntu-latest + steps: + - name: โฌ‡๏ธ Checkout + uses: actions/checkout@v7.0.0 + with: + ref: main + + - name: ๐Ÿ”ง Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: ๐Ÿ“ฆ Install update dependencies + run: bash scripts/github/update/install_python.sh + + - name: ๐Ÿ”„ Update pinned Ansible collections + run: bash scripts/github/update/galaxy_pins.sh + + - name: ๐Ÿ”Ž Check for changes + id: diff + run: bash scripts/github/resolve/check_git_changes.sh requirements + + - name: โš™๏ธ Generate app token + if: steps.diff.outputs.changed == 'true' + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.BOT_APP_CLIENT_ID }} + private-key: ${{ secrets.BOT_APP_PRIVATE_KEY }} + + - name: ๐Ÿ“‚ Open PR if collection pins changed + if: steps.diff.outputs.changed == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + UPDATE_BRANCH_PREFIX: update/galaxy-pins + UPDATE_COMMIT_MESSAGE: "update(galaxy): bump pinned Ansible collections" + UPDATE_PR_TITLE: "update(galaxy): bump pinned Ansible collections" + UPDATE_PR_BODY: "Automated update of the collection `version:` pins in `requirements/`. Both the Galaxy file and the git fallback are bumped together; upstream releases resolved via `git ls-remote --tags`." + UPDATE_BASE_BRANCH: main + run: bash scripts/github/update/open/pr.sh requirements + update-repository-refs: name: ๐ŸŒฟ Update repository refs if: vars.CI_ENABLE_AUTO_UPDATES == 'true' diff --git a/cli/contributing/update/galaxy/__init__.py b/cli/contributing/update/galaxy/__init__.py new file mode 100644 index 0000000000..f79db0b735 --- /dev/null +++ b/cli/contributing/update/galaxy/__init__.py @@ -0,0 +1,5 @@ +"""CLI entry point package for pinned Ansible collection updates.""" + +from pathlib import Path + +PROJECT_ROOT: Path = Path(__file__).resolve().parents[4] diff --git a/cli/contributing/update/galaxy/__main__.py b/cli/contributing/update/galaxy/__main__.py new file mode 100755 index 0000000000..c83fb083be --- /dev/null +++ b/cli/contributing/update/galaxy/__main__.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +from pathlib import Path + +from utils.update.galaxy import apply_updates, find_outdated_updates + +from . import PROJECT_ROOT + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Update pinned Ansible collection versions in requirements/ to their " + "latest upstream release tag." + ) + ) + parser.add_argument( + "--repo-root", + type=Path, + default=PROJECT_ROOT, + help="Repository root to scan and update", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print planned updates without modifying files", + ) + args = parser.parse_args() + + repo_root = args.repo_root.resolve() + updates = find_outdated_updates(repo_root) + + if not updates: + print("Pinned Ansible collections are already up to date.") + return 0 + + print("Planned collection pin updates:") + for update in updates: + print(f"- {update.pin.fqcn} {update.pin.version} -> {update.latest}") + + if args.dry_run: + return 0 + + changed_paths = apply_updates(repo_root, updates) + if changed_paths: + print("\nUpdated files:") + for path in changed_paths: + print(f"- {path.relative_to(repo_root)}") + else: + print("\nNo files changed.") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/github/update/galaxy_pins.sh b/scripts/github/update/galaxy_pins.sh new file mode 100755 index 0000000000..46201fd1d2 --- /dev/null +++ b/scripts/github/update/galaxy_pins.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd)" + +cd "${REPO_ROOT}" +make dotenv >/dev/null +set -a +# shellcheck disable=SC1091 +. .env +set +a +python3 -m cli.contributing.update.galaxy "$@" diff --git a/tests/unit/python/utils/update/test_galaxy.py b/tests/unit/python/utils/update/test_galaxy.py new file mode 100644 index 0000000000..3d977818cc --- /dev/null +++ b/tests/unit/python/utils/update/test_galaxy.py @@ -0,0 +1,134 @@ +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from utils.update.galaxy import apply_updates, collect_pins, find_outdated_updates + +GALAXY = """--- +collections: + - name: community.general + version: 13.2.0 + - name: kewlfft.aur + version: 0.13.0 +""" + +GIT = """--- +collections: + - name: community.general + source: https://github.com/ansible-collections/community.general.git + type: git + version: 13.2.0 + + - name: kewlfft.aur + source: https://github.com/kewlfft/ansible-aur.git + type: git + version: master +""" + + +def _repo(root: Path) -> Path: + (root / "requirements").mkdir() + (root / "requirements" / "requirements.galaxy.yml").write_text( + GALAXY, encoding="utf-8" + ) + (root / "requirements" / "requirements.git.yml").write_text(GIT, encoding="utf-8") + return root + + +class TestCollectPins(unittest.TestCase): + def test_only_collections_pinned_to_a_semver_in_both_files_are_collected(self): + with TemporaryDirectory() as tmp: + root = _repo(Path(tmp)) + + pins = collect_pins(root) + + self.assertEqual([pin.fqcn for pin in pins], ["community.general"]) + + def test_a_version_the_two_files_disagree_on_is_not_collected(self): + with TemporaryDirectory() as tmp: + root = _repo(Path(tmp)) + (root / "requirements" / "requirements.galaxy.yml").write_text( + GALAXY.replace("13.2.0", "13.4.0"), encoding="utf-8" + ) + + self.assertEqual(collect_pins(root), []) + + +class TestFindOutdatedUpdates(unittest.TestCase): + def test_a_higher_tag_of_the_same_depth_is_an_update(self): + with TemporaryDirectory() as tmp: + root = _repo(Path(tmp)) + with patch( + "utils.update.galaxy.git_ls_remote_tags", + return_value=["13.1.0", "13.2.0", "13.4.0"], + ): + updates = find_outdated_updates(root) + + self.assertEqual([u.latest for u in updates], ["13.4.0"]) + + def test_the_newest_tag_already_pinned_is_no_update(self): + with TemporaryDirectory() as tmp: + root = _repo(Path(tmp)) + with patch( + "utils.update.galaxy.git_ls_remote_tags", + return_value=["13.1.0", "13.2.0"], + ): + self.assertEqual(find_outdated_updates(root), []) + + def test_a_tag_of_a_different_depth_is_not_a_candidate(self): + with TemporaryDirectory() as tmp: + root = _repo(Path(tmp)) + with patch( + "utils.update.galaxy.git_ls_remote_tags", + return_value=["13.2.0", "14"], + ): + self.assertEqual(find_outdated_updates(root), []) + + +class TestApplyUpdates(unittest.TestCase): + def test_both_requirement_files_are_bumped_together(self): + with TemporaryDirectory() as tmp: + root = _repo(Path(tmp)) + with patch( + "utils.update.galaxy.git_ls_remote_tags", + return_value=["13.4.0"], + ): + updates = find_outdated_updates(root) + changed = apply_updates(root, updates) + + self.assertEqual(len(changed), 2) + galaxy_text = (root / "requirements" / "requirements.galaxy.yml").read_text( + encoding="utf-8" + ) + git_text = (root / "requirements" / "requirements.git.yml").read_text( + encoding="utf-8" + ) + self.assertIn("version: 13.4.0", galaxy_text) + self.assertIn("version: 13.4.0", git_text) + self.assertNotIn("13.2.0", galaxy_text) + self.assertNotIn("13.2.0", git_text) + + def test_an_entry_outside_the_update_keeps_its_version(self): + with TemporaryDirectory() as tmp: + root = _repo(Path(tmp)) + with patch( + "utils.update.galaxy.git_ls_remote_tags", + return_value=["13.4.0"], + ): + apply_updates(root, find_outdated_updates(root)) + + git_text = (root / "requirements" / "requirements.git.yml").read_text( + encoding="utf-8" + ) + self.assertIn("version: master", git_text) + self.assertIn( + "version: 0.13.0", + (root / "requirements" / "requirements.galaxy.yml").read_text( + encoding="utf-8" + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/update/galaxy.py b/utils/update/galaxy.py new file mode 100644 index 0000000000..b06a7714ae --- /dev/null +++ b/utils/update/galaxy.py @@ -0,0 +1,135 @@ +"""Pinned Ansible collection versions in ``requirements/``. + +Both requirement files name the same collections at the same versions: the +Galaxy file is the normal source, the git file the fallback used when +galaxy.ansible.com is unreachable. A bump that touches only one of them +would make the fallback install something the pin does not describe, so an +update is applied to both or to neither. + +Upstream releases are resolved from the git file's own ``source:`` URL via +``git ls-remote --tags``, not from the Galaxy API, because the collections +are tagged in the repository the fallback already clones. + +An entry whose ``version:`` is not a semver (a branch name, say) states an +intent this cannot second-guess and is skipped. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +import yaml + +from utils.update.base import is_semver, version_key +from utils.update.repository import git_ls_remote_tags + +GALAXY_REQUIREMENTS = Path("requirements/requirements.galaxy.yml") +GIT_REQUIREMENTS = Path("requirements/requirements.git.yml") + + +@dataclass(frozen=True) +class CollectionPin: + """One collection pinned in both requirement files. + + Args: + fqcn: fully qualified collection name, namespace and name. + version: the version both files pin today. + source: upstream git URL taken from the git requirements file. + """ + + fqcn: str + version: str + source: str + + +@dataclass(frozen=True) +class CollectionPinUpdate: + pin: CollectionPin + latest: str + + +def _entries(path: Path) -> list[dict]: + declared = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return [e for e in (declared.get("collections") or []) if isinstance(e, dict)] + + +def collect_pins(repo_root: Path) -> list[CollectionPin]: + """Collections pinned to the same semver in both requirement files. + + Args: + repo_root: repository root holding the ``requirements`` directory. + """ + galaxy = { + str(e.get("name")): str(e.get("version")) + for e in _entries(repo_root / GALAXY_REQUIREMENTS) + if e.get("name") and e.get("version") + } + pins: list[CollectionPin] = [] + for entry in _entries(repo_root / GIT_REQUIREMENTS): + fqcn = str(entry.get("name", "")) + version = str(entry.get("version", "")) + source = str(entry.get("source", "")) + if not (fqcn and source and is_semver(version)): + continue + if galaxy.get(fqcn) != version: + continue + pins.append(CollectionPin(fqcn=fqcn, version=version, source=source)) + return pins + + +def _highest_release(source: str, current: str) -> str | None: + candidates = [ + tag + for tag in git_ls_remote_tags(source) + if is_semver(tag) and tag.lstrip("v").count(".") == current.count(".") + ] + if not candidates: + return None + newest = max(candidates, key=version_key) + if version_key(newest) <= version_key(current): + return None + return newest.lstrip("v") + + +def find_outdated_updates(repo_root: Path) -> list[CollectionPinUpdate]: + """Pins whose upstream carries a higher release tag. + + Args: + repo_root: repository root holding the ``requirements`` directory. + """ + updates: list[CollectionPinUpdate] = [] + for pin in collect_pins(repo_root): + latest = _highest_release(pin.source, pin.version) + if latest: + updates.append(CollectionPinUpdate(pin=pin, latest=latest)) + return updates + + +def _rewrite(path: Path, fqcn: str, old: str, new: str) -> bool: + text = path.read_text(encoding="utf-8") + pattern = re.compile( + rf"(name:\s*{re.escape(fqcn)}\b(?:\n(?!\s*-\s).*)*?\n\s*version:\s*){re.escape(old)}\b" + ) + replaced, count = pattern.subn(rf"\g<1>{new}", text) + if not count: + return False + path.write_text(replaced, encoding="utf-8") + return True + + +def apply_updates(repo_root: Path, updates: list[CollectionPinUpdate]) -> list[Path]: + """Write every update into both requirement files. + + Args: + repo_root: repository root holding the ``requirements`` directory. + updates: the bumps to apply, as returned by :func:`find_outdated_updates`. + """ + changed: set[Path] = set() + for update in updates: + for relative in (GALAXY_REQUIREMENTS, GIT_REQUIREMENTS): + path = repo_root / relative + if _rewrite(path, update.pin.fqcn, update.pin.version, update.latest): + changed.add(path) + return sorted(changed) From 623ecc024867fb65953d14f2f07880e49b8f1cab Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 20:59:45 +0200 Subject: [PATCH 081/197] feat(update): retune the preferred collection source from measured latency Which of the two sources is faster is not a constant. galaxy.ansible.com degraded from 207s to 687s over one day and then served 502s outright, while the pinned git fallback carried the same install in 31s; a week later the order may be the other way round. The daily job times both and swaps the default only when the preferred one's median reaches three times the other's over twelve samples. A failed install is recorded as None and ranks worse than any duration, so an outage counts instead of dropping out of the statistic. Below a full window nothing changes, which keeps a single bad day from flipping the default. History lives in an artifact rather than the Actions cache: a schedule run resolves to the default branch and has read-only cache access there, so it could not carry the window forward. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/cron-collection-source.yml | 80 +++++++++ .../update/collection_source/__init__.py | 5 + .../update/collection_source/__main__.py | 152 ++++++++++++++++++ scripts/github/update/collection_source.sh | 13 ++ .../update/restore_collection_history.sh | 38 +++++ .../utils/update/test_collection_source.py | 114 +++++++++++++ utils/update/collection_source.py | 114 +++++++++++++ 7 files changed, 516 insertions(+) create mode 100644 .github/workflows/cron-collection-source.yml create mode 100644 cli/contributing/update/collection_source/__init__.py create mode 100755 cli/contributing/update/collection_source/__main__.py create mode 100755 scripts/github/update/collection_source.sh create mode 100644 scripts/github/update/restore_collection_history.sh create mode 100644 tests/unit/python/utils/update/test_collection_source.py create mode 100644 utils/update/collection_source.py diff --git a/.github/workflows/cron-collection-source.yml b/.github/workflows/cron-collection-source.yml new file mode 100644 index 0000000000..9d0f374db5 --- /dev/null +++ b/.github/workflows/cron-collection-source.yml @@ -0,0 +1,80 @@ +name: "โš–๏ธ Benchmark collection sources" +run-name: "${{ github.event_name == 'schedule' && 'โฐ ' || '' }}โš–๏ธ Benchmark collection sources" + +on: + schedule: + - cron: "0 3 * * *" + workflow_dispatch: + +concurrency: + group: collection-source-${{ github.repository }} + cancel-in-progress: true + +permissions: + contents: read + actions: read + +jobs: + benchmark-collection-sources: + name: โš–๏ธ Time Galaxy against the git fallback + if: vars.CI_ENABLE_AUTO_UPDATES == 'true' + runs-on: ubuntu-latest + env: + HISTORY_PATH: build/collection-source-history.json + steps: + - name: โฌ‡๏ธ Checkout + uses: actions/checkout@v7.0.0 + with: + ref: main + + - name: ๐Ÿ”ง Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: ๐Ÿ“ฆ Install project with the deploy extra + env: + PYTHON: python3 + run: bash scripts/install/python.sh deploy + + - name: โ™ป๏ธ Restore the sample history from the last run + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HISTORY_WORKFLOW: cron-collection-source.yml + run: bash scripts/github/update/restore_collection_history.sh + + - name: โฑ๏ธ Time both collection sources + run: bash scripts/github/update/collection_source.sh --history "${HISTORY_PATH}" + + - name: ๐Ÿ’พ Keep the sample history for the next run + uses: actions/upload-artifact@v7 + with: + name: collection-source-history + path: ${{ env.HISTORY_PATH }} + retention-days: 90 + + - name: ๐Ÿ”Ž Check for changes + id: diff + run: bash scripts/github/resolve/check_git_changes.sh default.env + + - name: โš™๏ธ Generate app token + if: steps.diff.outputs.changed == 'true' + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.BOT_APP_CLIENT_ID }} + private-key: ${{ secrets.BOT_APP_PRIVATE_KEY }} + + - name: ๐Ÿ“‚ Open PR if the preferred source changed + if: steps.diff.outputs.changed == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + UPDATE_BRANCH_PREFIX: update/collection-source + UPDATE_COMMIT_MESSAGE: "update(galaxy): retune the preferred collection source" + UPDATE_PR_TITLE: "update(galaxy): retune the preferred collection source" + UPDATE_PR_BODY: "Automated retune of `INFINITO_ANSIBLE_COLLECTIONS_SOURCE` in `default.env`. The preferred source's median install time reached at least 3x the other's over the last 12 daily samples; a failed install counts as worse than any duration." + UPDATE_BASE_BRANCH: main + run: bash scripts/github/update/open/pr.sh default.env diff --git a/cli/contributing/update/collection_source/__init__.py b/cli/contributing/update/collection_source/__init__.py new file mode 100644 index 0000000000..ab2c0b6137 --- /dev/null +++ b/cli/contributing/update/collection_source/__init__.py @@ -0,0 +1,5 @@ +"""CLI entry point package for retuning the preferred collection source.""" + +from pathlib import Path + +PROJECT_ROOT: Path = Path(__file__).resolve().parents[4] diff --git a/cli/contributing/update/collection_source/__main__.py b/cli/contributing/update/collection_source/__main__.py new file mode 100755 index 0000000000..ba5ed513bf --- /dev/null +++ b/cli/contributing/update/collection_source/__main__.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import time +from pathlib import Path +from tempfile import TemporaryDirectory + +from utils.update.collection_source import ( + RATIO, + SOURCES, + WINDOW, + append_sample, + decide, + load_history, +) + +from . import PROJECT_ROOT + +SETTING = "INFINITO_ANSIBLE_COLLECTIONS_SOURCE" +REQUIREMENTS = { + "galaxy": Path("requirements/requirements.galaxy.yml"), + "git": Path("requirements/requirements.git.yml"), +} + + +def read_preferred(env_file: Path) -> str: + """The source currently tried first, as declared in default.env. + + Args: + env_file: the default.env holding the setting. + """ + match = re.search( + rf"^{SETTING}=(\S+)\s*$", env_file.read_text(encoding="utf-8"), re.MULTILINE + ) + if match is None: + raise SystemExit(f"{SETTING} is not declared in {env_file}") + return match.group(1) + + +def write_preferred(env_file: Path, source: str) -> None: + """Point the setting at SOURCE. + + Args: + env_file: the default.env holding the setting. + source: the source to try first from now on. + """ + text = env_file.read_text(encoding="utf-8") + env_file.write_text( + re.sub(rf"^{SETTING}=\S+$", f"{SETTING}={source}", text, flags=re.MULTILINE), + encoding="utf-8", + ) + + +def measure(repo_root: Path, requirements: Path, timeout: int) -> float | None: + """Seconds a cold install from REQUIREMENTS takes, or None when it fails. + + Args: + repo_root: repository root the requirements path is relative to. + requirements: the requirements file to install from. + timeout: seconds after which the install counts as failed. + """ + with TemporaryDirectory() as target: + started = time.monotonic() + try: + completed = subprocess.run( + [ + sys.executable, + "-m", + "ansible.cli.galaxy", + "collection", + "install", + "-r", + str(repo_root / requirements), + "-p", + target, + "--force-with-deps", + ], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired: + return None + elapsed = time.monotonic() - started + return elapsed if completed.returncode == 0 else None + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Time both collection sources, record the sample, and retune the " + "preferred default when one source is durably slower." + ) + ) + parser.add_argument("--repo-root", type=Path, default=PROJECT_ROOT) + parser.add_argument( + "--history", + type=Path, + default=None, + help="JSON file holding the rolling window of samples", + ) + parser.add_argument( + "--timeout", + type=int, + default=900, + help="Seconds after which one install counts as failed", + ) + args = parser.parse_args() + + repo_root = args.repo_root.resolve() + env_file = repo_root / "default.env" + history_file = ( + args.history or repo_root / "build" / "collection-source-history.json" + ) + + sample: dict[str, float | None] = {} + for source in SOURCES: + seconds = measure(repo_root, REQUIREMENTS[source], args.timeout) + sample[source] = seconds + print(f"{source}: {'failed' if seconds is None else f'{seconds:.1f}s'}") + + history = append_sample(load_history(history_file), sample) + history_file.parent.mkdir(parents=True, exist_ok=True) + history_file.write_text(json.dumps(history, indent=2), encoding="utf-8") + + preferred = read_preferred(env_file) + verdict = decide(history, preferred) + print( + f"samples: {verdict.samples}/{WINDOW}, " + f"medians: {verdict.medians}, preferred: {verdict.preferred}" + ) + + if not verdict.changed: + print("Preferred collection source stays unchanged.") + return 0 + + write_preferred(env_file, verdict.proposed) + print( + f"Preferred collection source {verdict.preferred} -> {verdict.proposed}: " + f"its median is at least {RATIO}x the other's over {WINDOW} samples." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/github/update/collection_source.sh b/scripts/github/update/collection_source.sh new file mode 100755 index 0000000000..d414094979 --- /dev/null +++ b/scripts/github/update/collection_source.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd)" + +cd "${REPO_ROOT}" +make dotenv >/dev/null +set -a +# shellcheck disable=SC1091 +. .env +set +a +python3 -m cli.contributing.update.collection_source "$@" diff --git a/scripts/github/update/restore_collection_history.sh b/scripts/github/update/restore_collection_history.sh new file mode 100644 index 0000000000..fb49bb4cc6 --- /dev/null +++ b/scripts/github/update/restore_collection_history.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GH_TOKEN:?Missing GH_TOKEN}" +: "${GITHUB_REPOSITORY:?Missing GITHUB_REPOSITORY}" +: "${HISTORY_PATH:?Missing HISTORY_PATH}" +: "${HISTORY_WORKFLOW:?Missing HISTORY_WORKFLOW}" + +ARTIFACT_NAME="${ARTIFACT_NAME:-collection-source-history}" + +mkdir -p "$(dirname "${HISTORY_PATH}")" + +run_id="$( + gh run list \ + --repo "${GITHUB_REPOSITORY}" \ + --workflow "${HISTORY_WORKFLOW}" \ + --status success \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId // empty' +)" + +if [[ -z "${run_id}" ]]; then + echo "No previous successful run; starting a fresh history." + exit 0 +fi + +tmp_dir="$(mktemp -d)" +if ! gh run download "${run_id}" \ + --repo "${GITHUB_REPOSITORY}" \ + --name "${ARTIFACT_NAME}" \ + --dir "${tmp_dir}"; then + echo "Run ${run_id} carries no ${ARTIFACT_NAME}; starting a fresh history." + exit 0 +fi + +cp "${tmp_dir}/$(basename "${HISTORY_PATH}")" "${HISTORY_PATH}" +echo "Restored history from run ${run_id}." diff --git a/tests/unit/python/utils/update/test_collection_source.py b/tests/unit/python/utils/update/test_collection_source.py new file mode 100644 index 0000000000..fa0075d497 --- /dev/null +++ b/tests/unit/python/utils/update/test_collection_source.py @@ -0,0 +1,114 @@ +import json +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from utils.update.collection_source import ( + RATIO, + WINDOW, + append_sample, + decide, + load_history, +) + + +def _samples(galaxy, git, count=WINDOW): + return [{"galaxy": galaxy, "git": git} for _ in range(count)] + + +class TestAppendSample(unittest.TestCase): + def test_the_window_never_grows_past_its_size(self): + history = _samples(10.0, 10.0, WINDOW) + + grown = append_sample(history, {"galaxy": 99.0, "git": 1.0}) + + self.assertEqual(len(grown), WINDOW) + self.assertEqual(grown[-1], {"galaxy": 99.0, "git": 1.0}) + + def test_the_oldest_sample_is_the_one_dropped(self): + history = [{"galaxy": float(i), "git": 1.0} for i in range(WINDOW)] + + grown = append_sample(history, {"galaxy": 99.0, "git": 1.0}) + + self.assertEqual(grown[0]["galaxy"], 1.0) + + +class TestDecide(unittest.TestCase): + def test_below_a_full_window_the_default_is_left_alone(self): + history = _samples(300.0, 10.0, WINDOW - 1) + + verdict = decide(history, "galaxy") + + self.assertFalse(verdict.changed) + self.assertEqual(verdict.samples, WINDOW - 1) + + def test_exactly_the_ratio_swaps_the_preferred_source(self): + history = _samples(30.0, 10.0) + + verdict = decide(history, "galaxy") + + self.assertTrue(verdict.changed) + self.assertEqual(verdict.proposed, "git") + + def test_just_under_the_ratio_keeps_the_preferred_source(self): + history = _samples(10.0 * RATIO - 0.1, 10.0) + + self.assertFalse(decide(history, "galaxy").changed) + + def test_a_failed_install_ranks_worse_than_any_duration(self): + history = _samples(None, 10.0) + + verdict = decide(history, "galaxy") + + self.assertTrue(verdict.changed) + self.assertEqual(verdict.proposed, "git") + self.assertEqual(verdict.medians["galaxy"], float("inf")) + + def test_the_faster_source_is_never_swapped_away_from(self): + history = _samples(10.0, 300.0) + + self.assertFalse(decide(history, "galaxy").changed) + + def test_both_sources_failing_changes_nothing(self): + history = _samples(None, None) + + self.assertFalse(decide(history, "galaxy").changed) + + def test_a_single_outlier_does_not_move_the_median(self): + history = _samples(10.0, 10.0, WINDOW - 1) + history = append_sample(history, {"galaxy": 9000.0, "git": 10.0}) + + self.assertFalse(decide(history, "galaxy").changed) + + def test_the_verdict_is_taken_from_the_preferred_side(self): + history = _samples(10.0, 30.0) + + verdict = decide(history, "git") + + self.assertTrue(verdict.changed) + self.assertEqual(verdict.proposed, "galaxy") + + +class TestLoadHistory(unittest.TestCase): + def test_a_missing_file_reads_as_an_empty_history(self): + with TemporaryDirectory() as tmp: + self.assertEqual(load_history(Path(tmp) / "absent.json"), []) + + def test_a_corrupt_file_reads_as_an_empty_history(self): + with TemporaryDirectory() as tmp: + path = Path(tmp) / "history.json" + path.write_text("{not json", encoding="utf-8") + + self.assertEqual(load_history(path), []) + + def test_a_written_history_round_trips(self): + with TemporaryDirectory() as tmp: + path = Path(tmp) / "history.json" + history = _samples(1.0, 2.0, 3) + path.write_text(json.dumps(history), encoding="utf-8") + + self.assertEqual(load_history(path), history) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/update/collection_source.py b/utils/update/collection_source.py new file mode 100644 index 0000000000..1e28db2c9e --- /dev/null +++ b/utils/update/collection_source.py @@ -0,0 +1,114 @@ +"""Which requirements source ``scripts/install/ansible.sh`` should try first. + +Both sources carry the same pinned collections, so the only thing separating +them is how long they take and whether they answer at all. A single slow day +says nothing: galaxy.ansible.com degrades and recovers, and a git clone can +stall on one runner. The default is therefore retuned only when a rolling +window of daily samples shows one source durably behind the other. + +A sample whose install failed is recorded as ``None`` and ranks worse than +any duration, so an outage counts fully rather than being dropped. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from statistics import median +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + +SOURCES = ("galaxy", "git") +WINDOW = 12 +"""Samples the median is taken over; below this the default is left alone.""" + +RATIO = 3.0 +"""How many times slower the preferred source must be before it is swapped.""" + + +@dataclass(frozen=True) +class Verdict: + """The outcome of comparing the two sources. + + Args: + preferred: the source in force when the comparison ran. + proposed: the source the samples favour, equal to preferred when + nothing should change. + medians: median seconds per source, infinite when every sample failed. + samples: how many samples the medians were taken over. + """ + + preferred: str + proposed: str + medians: dict[str, float] + samples: int + + @property + def changed(self) -> bool: + return self.proposed != self.preferred + + +def load_history(path: Path) -> list[dict[str, float | None]]: + """Samples recorded so far, oldest first, or an empty list. + + Args: + path: JSON file holding the rolling window. + """ + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return [] + return payload if isinstance(payload, list) else [] + + +def append_sample( + history: list[dict[str, float | None]], sample: dict[str, float | None] +) -> list[dict[str, float | None]]: + """The history with SAMPLE appended, truncated to the newest WINDOW entries. + + Args: + history: samples recorded so far, oldest first. + sample: seconds per source for this run, None where the install failed. + """ + return [*history, sample][-WINDOW:] + + +def _median_seconds(history: list[dict[str, float | None]], source: str) -> float: + values = [ + float("inf") if entry.get(source) is None else float(entry[source]) + for entry in history + ] + return median(values) if values else float("inf") + + +def decide(history: list[dict[str, float | None]], preferred: str) -> Verdict: + """Whether the samples justify swapping the preferred source. + + Args: + history: samples recorded so far, oldest first. + preferred: the source currently tried first. + """ + medians = {source: _median_seconds(history, source) for source in SOURCES} + verdict = Verdict( + preferred=preferred, + proposed=preferred, + medians=medians, + samples=len(history), + ) + if len(history) < WINDOW: + return verdict + + other = next(source for source in SOURCES if source != preferred) + mine, theirs = medians[preferred], medians[other] + if theirs == float("inf") or theirs <= 0: + return verdict + if mine >= theirs * RATIO: + return Verdict( + preferred=preferred, + proposed=other, + medians=medians, + samples=len(history), + ) + return verdict From 7b37a7970fad4e736e5ed98822b4536fffe93966 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 21:04:15 +0200 Subject: [PATCH 082/197] fix(image): let a failed cache export fail the build ignore-error traded a loud fault for a silent one. The cache writes to the same GHCR repository as the image and under the same packages:write token, so every shared failure mode -- permissions, outage, rate limit -- already fails --push. What remains is buildkit's own cache manifest type, which a registry can reject while accepting an ordinary image push. That case is exactly the one worth hearing about. Swallowed, it leaves CI green while the cache never fills and every build silently pays the full 855-899s again; raised, the first run answers it once and for good. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/image/push.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/image/push.sh b/scripts/image/push.sh index d6c2f90cc0..b1a35b0c5a 100755 --- a/scripts/image/push.sh +++ b/scripts/image/push.sh @@ -49,7 +49,7 @@ while true; do --build-arg "INFINITO_DOCKER_CLI_INSTALL_SCRIPT=${INFINITO_DOCKER_CLI_INSTALL_SCRIPT:?source scripts/meta/env/load.sh}" \ "${nix_arg[@]}" \ --cache-from "type=registry,ref=${cache_ref}" \ - --cache-to "type=registry,mode=max,ignore-error=true,ref=${cache_ref}" \ + --cache-to "type=registry,mode=max,ref=${cache_ref}" \ "${BUILD_CONTEXT_DIR}"; then echo "Build & push succeeded on attempt ${attempt}/${max_attempts}." break From 466865945b544bc013ee948453e38d448689a4f9 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 21:24:58 +0200 Subject: [PATCH 083/197] perf(ci): build every distro image concurrently on one runner The matrix spent five runners on work that is almost entirely network wait: the collection install measured 0% CPU for its whole duration, and the parent-image pull, the package managers and the push are no different. One runner carries all five concurrently and frees the other four. Losing the matrix loses fail-fast: false with it, so push_all.sh awaits every build and reports each one by name before exiting non-zero. A broken distro no longer hides the other four, which is what made today's triage find three causes instead of one. Starts are spread by fibonacci seconds because each build opens with a parent-image pull of several hundred MB. Per-distro logs are grouped in the job output and uploaded as an artifact, and the pressure report now carries free -h: memory is the one resource this consolidation contends for, and an OOM kill would otherwise read as a build error. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/call-images-build-ci.yml | 22 +++-- scripts/github/runner/disk_pressure_report.sh | 2 + scripts/image/push_all.sh | 82 +++++++++++++++++++ 3 files changed, 97 insertions(+), 9 deletions(-) create mode 100755 scripts/image/push_all.sh diff --git a/.github/workflows/call-images-build-ci.yml b/.github/workflows/call-images-build-ci.yml index 1719d9a623..52888eddd7 100644 --- a/.github/workflows/call-images-build-ci.yml +++ b/.github/workflows/call-images-build-ci.yml @@ -58,18 +58,13 @@ jobs: run: scripts/github/runner/images_build_outputs.sh "${{ inputs.distros }}" "${{ inputs.image_tag }}" build: - name: ๐Ÿ—๏ธ Build (${{ matrix.distro }}) + name: ๐Ÿ’ฟ Image needs: prepare runs-on: ubuntu-latest permissions: contents: read packages: write - strategy: - fail-fast: false - matrix: - distro: ${{ fromJson(needs.prepare.outputs.matrix) }} - steps: - name: โฌ‡๏ธ Checkout trusted workflow repo (helpers) uses: actions/checkout@v7.0.0 @@ -128,19 +123,28 @@ jobs: set -euo pipefail echo "owner=$(scripts/meta/resolve/repository/owner.sh)" >> "$GITHUB_OUTPUT" - - name: ๐Ÿ”จ Build & push (retry 7x, 20s) + - name: ๐Ÿ”จ Build & push every distro (retry 7x, 20s) if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) }} shell: bash env: GITHUB_REPOSITORY_OWNER: ${{ steps.owner.outputs.owner }} BUILD_CONTEXT_DIR: _src - MATRIX_DISTRO: ${{ matrix.distro }} + INFINITO_DISTROS: ${{ join(fromJson(needs.prepare.outputs.matrix), ' ') }} IMAGE_TAG: ${{ needs.prepare.outputs.image_tag }} USE_NIX_TOKEN: ${{ !(github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) }} NIX_GITHUB_TOKEN: ${{ github.token }} MAX_ATTEMPTS: "7" RETRY_DELAY_SECONDS: "20" - run: ./scripts/image/push.sh + run: ./scripts/image/push_all.sh + + - name: ๐Ÿ“ค Upload per-distro build logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: image-build-logs + path: build/image-logs/ + retention-days: 7 + if-no-files-found: ignore - name: ๐Ÿ’ฝ Disk pressure (post-build) if: always() diff --git a/scripts/github/runner/disk_pressure_report.sh b/scripts/github/runner/disk_pressure_report.sh index 0c7e0ae233..a4b1aef1cb 100755 --- a/scripts/github/runner/disk_pressure_report.sh +++ b/scripts/github/runner/disk_pressure_report.sh @@ -14,6 +14,8 @@ echo "Runner: $(uname -a)" echo df -h echo +free -h +echo docker version || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error echo docker system df || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error diff --git a/scripts/image/push_all.sh b/scripts/image/push_all.sh new file mode 100755 index 0000000000..b365fa7dba --- /dev/null +++ b/scripts/image/push_all.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Argv: [ ...] (falls back to INFINITO_DISTROS) + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "${script_dir}/../.." && pwd)" + +distros=("$@") +if ((${#distros[@]} == 0)); then + : "${INFINITO_DISTROS:?Missing INFINITO_DISTROS and no distro arguments}" + read -r -a distros <<<"${INFINITO_DISTROS}" +fi + +log_dir="${INFINITO_BUILD_LOG_DIR:-${repo_root}/build/image-logs}" +mkdir -p "${log_dir}" + +group_open() { + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then + echo "::group::$1" + else + echo "โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ $1 โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" + fi +} + +group_close() { + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then + echo "::endgroup::" + fi +} + +# shellcheck source=scripts/meta/env/load.sh +source "${repo_root}/scripts/meta/env/load.sh" + +echo "๐Ÿ—๏ธ Building ${#distros[@]} image(s) concurrently: ${distros[*]}" +echo " Logs: ${log_dir}" + +gap_prev=1 +gap=2 + +pids=() +started=0 +for distro in "${distros[@]}"; do + if ((started > 0)); then + sleep "${gap}" + gap_next=$((gap_prev + gap)) + gap_prev="${gap}" + gap="${gap_next}" + fi + + MATRIX_DISTRO="${distro}" \ + bash "${script_dir}/push.sh" >"${log_dir}/${distro}.log" 2>&1 & + pids+=("$!") + echo " โ†’ ${distro} started as pid $!" + started=$((started + 1)) +done + +failed=() +index=0 +for pid in "${pids[@]}"; do + distro="${distros[${index}]}" + if wait "${pid}"; then + echo "โœ… ${distro}" + else + echo "โŒ ${distro}" + failed+=("${distro}") + fi + index=$((index + 1)) +done + +for distro in "${distros[@]}"; do + group_open "${distro}" + cat "${log_dir}/${distro}.log" || true # nocheck: shell-or-true -- a missing log must not mask the build verdict below + group_close +done + +if ((${#failed[@]} > 0)); then + echo "โŒ Failed distros: ${failed[*]}" + exit 1 +fi + +echo "๐ŸŽ‰ All ${#distros[@]} image(s) built and pushed" From 0aa314bb9b431364ffd14a6b8fde70bd40b95428 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 21:33:52 +0200 Subject: [PATCH 084/197] fix(install): stop preparing a venv parent that already holds a venv The parent directory was created and chowned on every run, before the script looked at whether the virtualenv exists. Where the parent is not writable that reached for sudo, which a sandboxed or no-new-privileges container refuses -- so an install that had nothing to do failed with 'The "no new privileges" flag is set'. Nothing needs preparing once the interpreter is there, so the check moves ahead of the preparation and returns. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/install/venv.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/install/venv.sh b/scripts/install/venv.sh index 8dcebdf096..3cc78f52bb 100755 --- a/scripts/install/venv.sh +++ b/scripts/install/venv.sh @@ -43,6 +43,11 @@ install_venv() { echo "๐ŸŽฏ Venv python target : ${venv_python}" echo + if [[ -x "${venv_python}" ]]; then + echo "โ†’ Virtualenv already exists" + return 0 + fi + local venv_parent venv_parent="$(dirname "${VENV}")" if [[ -n "${VIRTUAL_ENV:-}" && "${VIRTUAL_ENV}" == "${VENV}" ]]; then @@ -56,13 +61,9 @@ install_venv() { sudo chown "${USER:-$(whoami)}" "${venv_parent}" fi - if [[ ! -x "${venv_python}" ]]; then - echo "โ†’ Creating virtualenv ${VENV}" - "${bootstrap_python}" -m venv "${VENV}" - echo "โœ… Virtualenv created" - else - echo "โ†’ Virtualenv already exists" - fi + echo "โ†’ Creating virtualenv ${VENV}" + "${bootstrap_python}" -m venv "${VENV}" + echo "โœ… Virtualenv created" } install_venv From 67f5398985b373321378360f46611d59e781e6fe Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 21:45:56 +0200 Subject: [PATCH 085/197] fix(install): keep the lint bootstrap off third-party imports The lint installer imports this module, and it runs on the runner's bare interpreter before a single dependency exists. Sharing the version check with it therefore handed the bootstrap a PyYAML import, which killed it: 'ModuleNotFoundError: No module named yaml' raised through utils/install/lint/__main__.py:35. Routing the read through utils.cache.yaml would not have helped, that imports yaml too. The requirements files are this repository's own and hold one shape, so the pins are read with two regexes and the standard library. A line that does not match the shape leaves the entry unpinned, and unpinned already counts as unsatisfied, so the parser errs towards installing rather than towards skipping. Quoted versions are unwrapped: yaml accepts version: "13.4.0" and a formatter may introduce it, and carrying the quotes into the comparison would silently defeat the check instead of failing. Co-Authored-By: Claude Opus 5 (1M context) --- .../python/utils/install/test_collections.py | 37 ++++++++++++- utils/install/collections.py | 54 ++++++++++++++----- 2 files changed, 78 insertions(+), 13 deletions(-) diff --git a/tests/unit/python/utils/install/test_collections.py b/tests/unit/python/utils/install/test_collections.py index ed581be0b7..572a80ebfe 100644 --- a/tests/unit/python/utils/install/test_collections.py +++ b/tests/unit/python/utils/install/test_collections.py @@ -3,7 +3,7 @@ import unittest from pathlib import Path from tempfile import TemporaryDirectory -from utils.install.collections import unsatisfied +from utils.install.collections import declared_pins, unsatisfied REQUIREMENTS = """--- collections: @@ -29,6 +29,41 @@ def _install(collections_dir: Path, fqcn: str, version: str) -> None: ) +class TestDeclaredPins(unittest.TestCase): + def test_a_quoted_version_is_read_without_its_quotes(self): + body = ( + "---\ncollections:\n" + ' - name: double.quoted\n version: "13.4.0"\n' + " - name: single.quoted\n version: '1.0.0'\n" + " - name: commented\n version: 2.2.2 # pinned\n" + ) + with TemporaryDirectory() as tmp: + root = Path(tmp) + requirements = _write_requirements(root, body) + + self.assertEqual( + declared_pins(requirements), + [ + ("double.quoted", "13.4.0"), + ("single.quoted", "1.0.0"), + ("commented", "2.2.2"), + ], + ) + + def test_an_entry_without_a_version_stays_unpinned(self): + body = "---\ncollections:\n - name: bare.entry\n" + with TemporaryDirectory() as tmp: + root = Path(tmp) + + self.assertEqual( + declared_pins(_write_requirements(root, body)), [("bare.entry", None)] + ) + + def test_a_missing_file_declares_nothing(self): + with TemporaryDirectory() as tmp: + self.assertEqual(declared_pins(Path(tmp) / "absent.yml"), []) + + class TestUnsatisfied(unittest.TestCase): def test_every_pin_installed_at_its_version_is_satisfied(self): with TemporaryDirectory() as tmp: diff --git a/utils/install/collections.py b/utils/install/collections.py index 338072cfd4..c6fd3cf6ba 100644 --- a/utils/install/collections.py +++ b/utils/install/collections.py @@ -1,12 +1,48 @@ -"""Compare the pinned Galaxy requirements against what is on disk.""" +"""Compare the pinned Galaxy requirements against what is on disk. + +Imported by the lint bootstrap, which runs on a bare interpreter before any +dependency is installed, so this module stays on the standard library. The +requirements files it reads are this repository's own and hold one shape: +a ``collections:`` list of ``- name:`` entries with an optional ``version:``. +A line that does not match that shape leaves the entry unpinned, which the +caller treats as unsatisfied and installs. +""" from __future__ import annotations import json +import re import sys from pathlib import Path -import yaml +_NAME = re.compile(r"^\s*-\s+name:\s*(?P[A-Za-z0-9_.]+)\s*(?:#.*)?$") +_VERSION = re.compile( + r"^\s*version:\s*(?P[\"']?)(?P[^\s#\"']+)(?P=quote)\s*(?:#.*)?$" +) + + +def declared_pins(requirements_file: Path) -> list[tuple[str, str | None]]: + """The ``(fqcn, version)`` pairs a requirements file declares, in order. + + Args: + requirements_file: the requirements.galaxy.yml to read. + """ + pins: list[tuple[str, str | None]] = [] + try: + lines = requirements_file.read_text(encoding="utf-8").splitlines() + except OSError: + return pins + + for line in lines: + name_match = _NAME.match(line) + if name_match: + pins.append((name_match.group("name"), None)) + continue + version_match = _VERSION.match(line) + if version_match and pins: + fqcn, _ = pins[-1] + pins[-1] = (fqcn, version_match.group("version")) + return pins def _installed_version(collections_dir: Path, namespace: str, name: str) -> str | None: @@ -38,19 +74,13 @@ def unsatisfied(requirements_file: Path, collections_dir: Path) -> list[str]: requirements_file: the requirements.galaxy.yml to read. collections_dir: the ``-p`` target ansible-galaxy installs into. """ - declared = yaml.safe_load(requirements_file.read_text(encoding="utf-8")) or {} missing: list[str] = [] - for entry in declared.get("collections") or []: - if not isinstance(entry, dict): - missing.append(str(entry)) - continue - fqcn = str(entry.get("name", "")) - pinned = entry.get("version") - if not fqcn or "." not in fqcn or not pinned: - missing.append(fqcn or repr(entry)) + for fqcn, pinned in declared_pins(requirements_file): + if "." not in fqcn or not pinned: + missing.append(fqcn) continue namespace, _, name = fqcn.partition(".") - if _installed_version(collections_dir, namespace, name) != str(pinned): + if _installed_version(collections_dir, namespace, name) != pinned: missing.append(f"{fqcn}:{pinned}") return missing From 94baeb750208a177ff60d864d84cb3b457475ffb Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 21:47:10 +0200 Subject: [PATCH 086/197] fix(lint): meet the repository rules the collection work broke The gate flagged six of my own changes. INFINITO_ANSIBLE_COLLECTIONS_SOURCE reached default.env without a handler, so the generator dropped it silently; ansible.sh then carried a shell-side default for the same key, a second source that drifts. Both are now one declaration, read as required, so a missing key fails loudly instead of falling back. File reads route through the session cache except where the same run rewrites the file, which is the documented opt-out and now says so per call. The updater reads yaml through utils.cache.yaml rather than calling safe_load itself. restore_collection_history.sh was committed without its executable bit, and the new workflow's name was 31 characters where the dispatch menu truncates at 25; the explanatory wording lives in run-name instead. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/cron-collection-source.yml | 2 +- .../update/collection_source/__main__.py | 6 +++-- .../update/restore_collection_history.sh | 0 scripts/install/ansible.sh | 4 ++- tests/unit/python/utils/update/test_galaxy.py | 25 +++++++------------ utils/env/handlers/passthrough.py | 1 + utils/update/collection_source.py | 2 +- utils/update/galaxy.py | 7 +++--- 8 files changed, 22 insertions(+), 25 deletions(-) mode change 100644 => 100755 scripts/github/update/restore_collection_history.sh diff --git a/.github/workflows/cron-collection-source.yml b/.github/workflows/cron-collection-source.yml index 9d0f374db5..50dc753b10 100644 --- a/.github/workflows/cron-collection-source.yml +++ b/.github/workflows/cron-collection-source.yml @@ -1,4 +1,4 @@ -name: "โš–๏ธ Benchmark collection sources" +name: "โš–๏ธ Source benchmark" run-name: "${{ github.event_name == 'schedule' && 'โฐ ' || '' }}โš–๏ธ Benchmark collection sources" on: diff --git a/cli/contributing/update/collection_source/__main__.py b/cli/contributing/update/collection_source/__main__.py index ba5ed513bf..2a7342ad92 100755 --- a/cli/contributing/update/collection_source/__main__.py +++ b/cli/contributing/update/collection_source/__main__.py @@ -35,7 +35,9 @@ def read_preferred(env_file: Path) -> str: env_file: the default.env holding the setting. """ match = re.search( - rf"^{SETTING}=(\S+)\s*$", env_file.read_text(encoding="utf-8"), re.MULTILINE + rf"^{SETTING}=(\S+)\s*$", + env_file.read_text(encoding="utf-8"), # nocheck: cache-read -- write_preferred rewrites this file in the same run + re.MULTILINE, ) if match is None: raise SystemExit(f"{SETTING} is not declared in {env_file}") @@ -49,7 +51,7 @@ def write_preferred(env_file: Path, source: str) -> None: env_file: the default.env holding the setting. source: the source to try first from now on. """ - text = env_file.read_text(encoding="utf-8") + text = env_file.read_text(encoding="utf-8") # nocheck: cache-read -- rewritten on the next line env_file.write_text( re.sub(rf"^{SETTING}=\S+$", f"{SETTING}={source}", text, flags=re.MULTILINE), encoding="utf-8", diff --git a/scripts/github/update/restore_collection_history.sh b/scripts/github/update/restore_collection_history.sh old mode 100644 new mode 100755 diff --git a/scripts/install/ansible.sh b/scripts/install/ansible.sh index 3f0152a001..6f5ce14ed8 100755 --- a/scripts/install/ansible.sh +++ b/scripts/install/ansible.sh @@ -25,7 +25,9 @@ if [[ -z "${INFINITO_ANSIBLE_COLLECTIONS_SOURCE:-}" ]]; then source <(grep -E '^INFINITO_ANSIBLE_COLLECTIONS_SOURCE=' default.env) fi -case "${INFINITO_ANSIBLE_COLLECTIONS_SOURCE:-galaxy}" in +: "${INFINITO_ANSIBLE_COLLECTIONS_SOURCE:?not declared in default.env}" + +case "${INFINITO_ANSIBLE_COLLECTIONS_SOURCE}" in galaxy) SOURCE_ORDER=("Galaxy:${GALAXY_REQ}" "Git:${GIT_REQ}") ;; git) SOURCE_ORDER=("Git:${GIT_REQ}" "Galaxy:${GALAXY_REQ}") ;; *) diff --git a/tests/unit/python/utils/update/test_galaxy.py b/tests/unit/python/utils/update/test_galaxy.py index 3d977818cc..06f7a2789f 100644 --- a/tests/unit/python/utils/update/test_galaxy.py +++ b/tests/unit/python/utils/update/test_galaxy.py @@ -27,6 +27,11 @@ collections: """ +def _read(root: Path, name: str) -> str: + # nocheck: cache-read -- apply_updates rewrote this file inside the same test + return (root / "requirements" / name).read_text(encoding="utf-8") + + def _repo(root: Path) -> Path: (root / "requirements").mkdir() (root / "requirements" / "requirements.galaxy.yml").write_text( @@ -98,12 +103,8 @@ class TestApplyUpdates(unittest.TestCase): changed = apply_updates(root, updates) self.assertEqual(len(changed), 2) - galaxy_text = (root / "requirements" / "requirements.galaxy.yml").read_text( - encoding="utf-8" - ) - git_text = (root / "requirements" / "requirements.git.yml").read_text( - encoding="utf-8" - ) + galaxy_text = _read(root, "requirements.galaxy.yml") + git_text = _read(root, "requirements.git.yml") self.assertIn("version: 13.4.0", galaxy_text) self.assertIn("version: 13.4.0", git_text) self.assertNotIn("13.2.0", galaxy_text) @@ -118,16 +119,8 @@ class TestApplyUpdates(unittest.TestCase): ): apply_updates(root, find_outdated_updates(root)) - git_text = (root / "requirements" / "requirements.git.yml").read_text( - encoding="utf-8" - ) - self.assertIn("version: master", git_text) - self.assertIn( - "version: 0.13.0", - (root / "requirements" / "requirements.galaxy.yml").read_text( - encoding="utf-8" - ), - ) + self.assertIn("version: master", _read(root, "requirements.git.yml")) + self.assertIn("version: 0.13.0", _read(root, "requirements.galaxy.yml")) if __name__ == "__main__": diff --git a/utils/env/handlers/passthrough.py b/utils/env/handlers/passthrough.py index f054fda41c..8d563d5875 100644 --- a/utils/env/handlers/passthrough.py +++ b/utils/env/handlers/passthrough.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: STATIC_KEYS = ( "INFINITO_SRC_DIR", "INFINITO_VENV_DIR", + "INFINITO_ANSIBLE_COLLECTIONS_SOURCE", "INFINITO_BIND_IP", "INFINITO_BUILD", "INFINITO_SUBNET", diff --git a/utils/update/collection_source.py b/utils/update/collection_source.py index 1e28db2c9e..eee2819253 100644 --- a/utils/update/collection_source.py +++ b/utils/update/collection_source.py @@ -57,7 +57,7 @@ def load_history(path: Path) -> list[dict[str, float | None]]: path: JSON file holding the rolling window. """ try: - payload = json.loads(path.read_text(encoding="utf-8")) + payload = json.loads(path.read_text(encoding="utf-8")) # nocheck: cache-read -- the history file is rewritten every run, a cached copy would go stale except (OSError, ValueError): return [] return payload if isinstance(payload, list) else [] diff --git a/utils/update/galaxy.py b/utils/update/galaxy.py index b06a7714ae..5473a2c3c1 100644 --- a/utils/update/galaxy.py +++ b/utils/update/galaxy.py @@ -20,8 +20,7 @@ import re from dataclasses import dataclass from pathlib import Path -import yaml - +from utils.cache.yaml import load_yaml_any from utils.update.base import is_semver, version_key from utils.update.repository import git_ls_remote_tags @@ -51,7 +50,7 @@ class CollectionPinUpdate: def _entries(path: Path) -> list[dict]: - declared = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + declared = load_yaml_any(str(path), default_if_missing={}) or {} return [e for e in (declared.get("collections") or []) if isinstance(e, dict)] @@ -108,7 +107,7 @@ def find_outdated_updates(repo_root: Path) -> list[CollectionPinUpdate]: def _rewrite(path: Path, fqcn: str, old: str, new: str) -> bool: - text = path.read_text(encoding="utf-8") + text = path.read_text(encoding="utf-8") # nocheck: cache-read -- rewritten below, a cached copy would go stale mid-run pattern = re.compile( rf"(name:\s*{re.escape(fqcn)}\b(?:\n(?!\s*-\s).*)*?\n\s*version:\s*){re.escape(old)}\b" ) From 7d8505c8c3bf91c40683ffeeb349b689cb8d667c Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 21:48:12 +0200 Subject: [PATCH 087/197] perf(ci): fold the matrix job into the image job and stream its output Computing the distro list no longer justifies a runner of its own: it is two steps, and the job that consumes them already checks the repository out. The workflow now occupies one runner instead of seven. The step also went silent for minutes at a time, printing five pids and then nothing until every build had finished. Each build's output is piped through a [distro][HH:MM:SS] stamp as it arrives, so the log shows what is happening while it happens; tee still files the per-distro logs for the artifact, and the replay block at the end is gone as duplication. A pipeline hands $! the pid of tee rather than of the build, which would have swallowed every failure, so each build runs in a subshell under pipefail and wait reads that. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/call-images-build-ci.yml | 28 ++++++----------- scripts/image/push_all.sh | 35 ++++++++-------------- 2 files changed, 22 insertions(+), 41 deletions(-) diff --git a/.github/workflows/call-images-build-ci.yml b/.github/workflows/call-images-build-ci.yml index 52888eddd7..227fd3cfb4 100644 --- a/.github/workflows/call-images-build-ci.yml +++ b/.github/workflows/call-images-build-ci.yml @@ -41,25 +41,8 @@ concurrency: cancel-in-progress: false jobs: - prepare: - name: ๐Ÿ“‹ Build Matrix - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.out.outputs.matrix }} - image_tag: ${{ steps.out.outputs.image_tag }} - steps: - - uses: actions/checkout@v7.0.0 - - - name: ๐Ÿ“ฆ Install jq - run: ./scripts/install/apt.sh jq - - - name: ๐Ÿงฎ Compute matrix JSON - id: out - run: scripts/github/runner/images_build_outputs.sh "${{ inputs.distros }}" "${{ inputs.image_tag }}" - build: name: ๐Ÿ’ฟ Image - needs: prepare runs-on: ubuntu-latest permissions: contents: read @@ -81,6 +64,13 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.allow_unsafe_checkout }} + - name: ๐Ÿ“ฆ Install jq + run: ./scripts/install/apt.sh jq + + - name: ๐Ÿงฎ Compute the distro list and image tag + id: out + run: scripts/github/runner/images_build_outputs.sh "${{ inputs.distros }}" "${{ inputs.image_tag }}" + - name: ๐Ÿ’ฝ Disk pressure (start) shell: bash run: bash scripts/github/runner/disk_pressure_report.sh START @@ -129,8 +119,8 @@ jobs: env: GITHUB_REPOSITORY_OWNER: ${{ steps.owner.outputs.owner }} BUILD_CONTEXT_DIR: _src - INFINITO_DISTROS: ${{ join(fromJson(needs.prepare.outputs.matrix), ' ') }} - IMAGE_TAG: ${{ needs.prepare.outputs.image_tag }} + INFINITO_DISTROS: ${{ join(fromJson(steps.out.outputs.matrix), ' ') }} + IMAGE_TAG: ${{ steps.out.outputs.image_tag }} USE_NIX_TOKEN: ${{ !(github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) }} NIX_GITHUB_TOKEN: ${{ github.token }} MAX_ATTEMPTS: "7" diff --git a/scripts/image/push_all.sh b/scripts/image/push_all.sh index b365fa7dba..11ca4b67b8 100755 --- a/scripts/image/push_all.sh +++ b/scripts/image/push_all.sh @@ -12,21 +12,15 @@ if ((${#distros[@]} == 0)); then read -r -a distros <<<"${INFINITO_DISTROS}" fi -log_dir="${INFINITO_BUILD_LOG_DIR:-${repo_root}/build/image-logs}" +log_dir="${BUILD_LOG_DIR:-${repo_root}/build/image-logs}" mkdir -p "${log_dir}" -group_open() { - if [[ -n "${GITHUB_ACTIONS:-}" ]]; then - echo "::group::$1" - else - echo "โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ $1 โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" - fi -} - -group_close() { - if [[ -n "${GITHUB_ACTIONS:-}" ]]; then - echo "::endgroup::" - fi +stamp() { + local distro="$1" + local line + while IFS= read -r line; do + printf '[%s][%(%H:%M:%S)T] %s\n' "${distro}" -1 "${line}" + done } # shellcheck source=scripts/meta/env/load.sh @@ -48,10 +42,13 @@ for distro in "${distros[@]}"; do gap="${gap_next}" fi - MATRIX_DISTRO="${distro}" \ - bash "${script_dir}/push.sh" >"${log_dir}/${distro}.log" 2>&1 & + ( + set -o pipefail + MATRIX_DISTRO="${distro}" bash "${script_dir}/push.sh" 2>&1 | + stamp "${distro}" | + tee "${log_dir}/${distro}.log" + ) & pids+=("$!") - echo " โ†’ ${distro} started as pid $!" started=$((started + 1)) done @@ -68,12 +65,6 @@ for pid in "${pids[@]}"; do index=$((index + 1)) done -for distro in "${distros[@]}"; do - group_open "${distro}" - cat "${log_dir}/${distro}.log" || true # nocheck: shell-or-true -- a missing log must not mask the build verdict below - group_close -done - if ((${#failed[@]} > 0)); then echo "โŒ Failed distros: ${failed[*]}" exit 1 From f0ecf3e33789b9d41f1fea0c9ce3a3146cd0760f Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 21:51:25 +0200 Subject: [PATCH 088/197] style(update): apply the formatting the gate produced The nocheck markers were written as long single lines; ruff splits the call across three lines and leaves the marker behind the closing paren. Committing the formatter's own output keeps the tree and the history in step. Co-Authored-By: Claude Opus 5 (1M context) --- cli/contributing/update/collection_source/__main__.py | 8 ++++++-- utils/update/collection_source.py | 4 +++- utils/update/galaxy.py | 4 +++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cli/contributing/update/collection_source/__main__.py b/cli/contributing/update/collection_source/__main__.py index 2a7342ad92..f0ffb7694e 100755 --- a/cli/contributing/update/collection_source/__main__.py +++ b/cli/contributing/update/collection_source/__main__.py @@ -36,7 +36,9 @@ def read_preferred(env_file: Path) -> str: """ match = re.search( rf"^{SETTING}=(\S+)\s*$", - env_file.read_text(encoding="utf-8"), # nocheck: cache-read -- write_preferred rewrites this file in the same run + env_file.read_text( + encoding="utf-8" + ), # nocheck: cache-read -- write_preferred rewrites this file in the same run re.MULTILINE, ) if match is None: @@ -51,7 +53,9 @@ def write_preferred(env_file: Path, source: str) -> None: env_file: the default.env holding the setting. source: the source to try first from now on. """ - text = env_file.read_text(encoding="utf-8") # nocheck: cache-read -- rewritten on the next line + text = env_file.read_text( + encoding="utf-8" + ) # nocheck: cache-read -- rewritten on the next line env_file.write_text( re.sub(rf"^{SETTING}=\S+$", f"{SETTING}={source}", text, flags=re.MULTILINE), encoding="utf-8", diff --git a/utils/update/collection_source.py b/utils/update/collection_source.py index eee2819253..921faf6457 100644 --- a/utils/update/collection_source.py +++ b/utils/update/collection_source.py @@ -57,7 +57,9 @@ def load_history(path: Path) -> list[dict[str, float | None]]: path: JSON file holding the rolling window. """ try: - payload = json.loads(path.read_text(encoding="utf-8")) # nocheck: cache-read -- the history file is rewritten every run, a cached copy would go stale + payload = json.loads( + path.read_text(encoding="utf-8") + ) # nocheck: cache-read -- the history file is rewritten every run, a cached copy would go stale except (OSError, ValueError): return [] return payload if isinstance(payload, list) else [] diff --git a/utils/update/galaxy.py b/utils/update/galaxy.py index 5473a2c3c1..a29a3edf1c 100644 --- a/utils/update/galaxy.py +++ b/utils/update/galaxy.py @@ -107,7 +107,9 @@ def find_outdated_updates(repo_root: Path) -> list[CollectionPinUpdate]: def _rewrite(path: Path, fqcn: str, old: str, new: str) -> bool: - text = path.read_text(encoding="utf-8") # nocheck: cache-read -- rewritten below, a cached copy would go stale mid-run + text = path.read_text( + encoding="utf-8" + ) # nocheck: cache-read -- rewritten below, a cached copy would go stale mid-run pattern = re.compile( rf"(name:\s*{re.escape(fqcn)}\b(?:\n(?!\s*-\s).*)*?\n\s*version:\s*){re.escape(old)}\b" ) From d4ae5cf9f02199bea40b904f71d2946ee31ad5d5 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 22:00:30 +0200 Subject: [PATCH 089/197] fix(lint): put the cache-read markers where the rule reads them suppressed_line_numbers matches the marker on the flagged line itself, and ruff had split two of the calls so the comment landed behind the closing paren, one line too low. The reads that can be cached now go through utils.cache.files, which the bare interpreter imports fine; the two that cannot keep a marker on a line short enough to stay whole. The parser also gains the two guards utils/install/lint/pinned.py already carries for the same bootstrap problem: its reading is cross-checked against PyYAML on both requirements files, and a subprocess with yaml made unimportable proves the module still works without it. That turns the manual venv check into a standing test. Co-Authored-By: Claude Opus 5 (1M context) --- .../python/utils/install/test_collections.py | 61 +++++++++++++++++++ tests/unit/python/utils/update/test_galaxy.py | 4 +- utils/install/collections.py | 6 +- utils/update/collection_source.py | 5 +- 4 files changed, 69 insertions(+), 7 deletions(-) diff --git a/tests/unit/python/utils/install/test_collections.py b/tests/unit/python/utils/install/test_collections.py index 572a80ebfe..7f552aa32e 100644 --- a/tests/unit/python/utils/install/test_collections.py +++ b/tests/unit/python/utils/install/test_collections.py @@ -1,10 +1,41 @@ import json +import subprocess +import sys import unittest from pathlib import Path from tempfile import TemporaryDirectory +from utils import PROJECT_ROOT +from utils.cache.yaml import load_yaml_any from utils.install.collections import declared_pins, unsatisfied +_REQUIREMENTS = ( + PROJECT_ROOT / "requirements" / "requirements.galaxy.yml", + PROJECT_ROOT / "requirements" / "requirements.git.yml", +) + +_BOOTSTRAP_PROBE = """ +import sys + + +class _NoPyYAML: + def find_spec(self, fullname, path=None, target=None): + if fullname == "yaml" or fullname.startswith("yaml."): + raise ImportError("PyYAML is absent during the lint bootstrap") + return None + + +sys.meta_path.insert(0, _NoPyYAML()) + +from pathlib import Path + +from utils.install.lint.ansible import collections as lint_collections +from utils.install.collections import declared_pins + +for fqcn, version in declared_pins(Path("requirements/requirements.galaxy.yml")): + print(fqcn, version) +""" + REQUIREMENTS = """--- collections: - name: community.general @@ -29,6 +60,36 @@ def _install(collections_dir: Path, fqcn: str, version: str) -> None: ) +class TestAgainstPyYAML(unittest.TestCase): + def test_the_hand_parser_agrees_with_pyyaml(self): + for path in _REQUIREMENTS: + with self.subTest(requirements=path.name): + declared = load_yaml_any(str(path), default_if_missing={}) or {} + expected = [ + (str(entry.get("name")), entry.get("version")) + for entry in declared.get("collections") or [] + if isinstance(entry, dict) and entry.get("name") + ] + expected = [ + (name, None if version is None else str(version)) + for name, version in expected + ] + + self.assertEqual(declared_pins(path), expected) + + def test_the_parser_runs_with_pyyaml_unimportable(self): + result = subprocess.run( + [sys.executable, "-c", _BOOTSTRAP_PROBE], + cwd=str(PROJECT_ROOT), + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("community.general", result.stdout) + + class TestDeclaredPins(unittest.TestCase): def test_a_quoted_version_is_read_without_its_quotes(self): body = ( diff --git a/tests/unit/python/utils/update/test_galaxy.py b/tests/unit/python/utils/update/test_galaxy.py index 06f7a2789f..c05a02d292 100644 --- a/tests/unit/python/utils/update/test_galaxy.py +++ b/tests/unit/python/utils/update/test_galaxy.py @@ -28,8 +28,8 @@ collections: def _read(root: Path, name: str) -> str: - # nocheck: cache-read -- apply_updates rewrote this file inside the same test - return (root / "requirements" / name).read_text(encoding="utf-8") + path = root / "requirements" / name + return path.read_text() # nocheck: cache-read -- rewritten by this test def _repo(root: Path) -> Path: diff --git a/utils/install/collections.py b/utils/install/collections.py index c6fd3cf6ba..08223a6616 100644 --- a/utils/install/collections.py +++ b/utils/install/collections.py @@ -15,6 +15,8 @@ import re import sys from pathlib import Path +from utils.cache.files import read_text + _NAME = re.compile(r"^\s*-\s+name:\s*(?P[A-Za-z0-9_.]+)\s*(?:#.*)?$") _VERSION = re.compile( r"^\s*version:\s*(?P[\"']?)(?P[^\s#\"']+)(?P=quote)\s*(?:#.*)?$" @@ -29,7 +31,7 @@ def declared_pins(requirements_file: Path) -> list[tuple[str, str | None]]: """ pins: list[tuple[str, str | None]] = [] try: - lines = requirements_file.read_text(encoding="utf-8").splitlines() + lines = read_text(str(requirements_file)).splitlines() except OSError: return pins @@ -57,7 +59,7 @@ def _installed_version(collections_dir: Path, namespace: str, name: str) -> str collections_dir / "ansible_collections" / namespace / name / "MANIFEST.json" ) try: - payload = json.loads(manifest.read_text(encoding="utf-8")) + payload = json.loads(read_text(str(manifest))) except (OSError, ValueError): return None version = payload.get("collection_info", {}).get("version") diff --git a/utils/update/collection_source.py b/utils/update/collection_source.py index 921faf6457..546ed3488e 100644 --- a/utils/update/collection_source.py +++ b/utils/update/collection_source.py @@ -57,9 +57,8 @@ def load_history(path: Path) -> list[dict[str, float | None]]: path: JSON file holding the rolling window. """ try: - payload = json.loads( - path.read_text(encoding="utf-8") - ) # nocheck: cache-read -- the history file is rewritten every run, a cached copy would go stale + raw = path.read_text() # nocheck: cache-read -- rewritten every run + payload = json.loads(raw) except (OSError, ValueError): return [] return payload if isinstance(payload, list) else [] From 5f6dedf7a8f75ce4abdd23e5fdd5a84a4f3003c8 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 22:04:28 +0200 Subject: [PATCH 090/197] perf(ci): stream the DNS run per distro instead of replaying it The job could sit silent for its full 30-minute budget and only then print five groups, so a hanging distro was indistinguishable from a slow one. Each container's output is stamped with [distro][HH:MM:SS] as it arrives, and the per-distro verdict is printed when that distro finishes rather than at the end. A pipeline hands $! the pid of tee rather than of the test, which would have turned every DNS failure into a green job, so each run goes through a subshell under pipefail. The ::error:: annotation, the per-distro logs and the container cleanup are unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/tests/dns/parallel.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/tests/dns/parallel.sh b/scripts/tests/dns/parallel.sh index 8e6d2162fb..49390a6fc5 100755 --- a/scripts/tests/dns/parallel.sh +++ b/scripts/tests/dns/parallel.sh @@ -43,9 +43,22 @@ run_one() { docker exec "${name}" bash -lc 'exec "${INFINITO_SRC_DIR}/scripts/tests/dns/inside.sh"' } +stamp() { + local distro="$1" + local line + while IFS= read -r line; do + printf '[%s][%(%H:%M:%S)T] %s\n' "${distro}" -1 "${line}" + done +} + +echo "๐ŸŒ Testing DNS on ${#distros[@]} distro(s) concurrently: ${distros[*]}" + declare -A pid for d in "${distros[@]}"; do - run_one "${d}" >"/tmp/dns-${d}.log" 2>&1 & + ( + set -o pipefail + run_one "${d}" 2>&1 | stamp "${d}" | tee "/tmp/dns-${d}.log" + ) & pid["${d}"]=$! done @@ -53,12 +66,12 @@ rc_total=0 for d in "${distros[@]}"; do rc=0 wait "${pid[${d}]}" || rc=$? - echo "::group::DNS ${d} (exit ${rc})" - cat "/tmp/dns-${d}.log" - echo "::endgroup::" if [[ "${rc}" -ne 0 ]]; then + echo "โŒ ${d} (exit ${rc})" echo "::error::DNS test failed for ${d}" rc_total=1 + else + echo "โœ… ${d}" fi docker rm -f "dns-dind-${d}" >/dev/null 2>&1 || true # nocheck: shell-or-true -- cleanup of a possibly already-removed container done From 2e5f70383ef49b9c85642deae6a228c85e7b46d0 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 22:07:58 +0200 Subject: [PATCH 091/197] perf(ci): stop the deploy chunks waiting on the DNS test The chunks gated on test-dns, so every deploy row sat behind a check that takes minutes and rarely breaks. They now wait on build-ci-images and validate-selection only, which is what they actually need: the images they deploy and the selection they deploy from. validate-selection moves into every chunk's needs, where so far only chunk 0 named it. The DNS test keeps its teeth. It still runs against the freshly built images, and the finish job fails the pipeline on any upstream failure or cancellation, so a broken resolver still turns the run red -- at the end rather than before the chunks, at the price of spending runner time on deploys that a cheap check would have pre-empted. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/call-orchestrator.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/call-orchestrator.yml b/.github/workflows/call-orchestrator.yml index 72d9f67507..b71a3b4879 100644 --- a/.github/workflows/call-orchestrator.yml +++ b/.github/workflows/call-orchestrator.yml @@ -161,14 +161,12 @@ jobs: && needs.lint.result == 'success' && needs.test.result == 'success' && needs.build-ci-images.result == 'success' - && needs.images-mirror-missing.result == 'success' - && needs.test-dns.result == 'success' }} + && needs.images-mirror-missing.result == 'success' }} needs: - lint - test - build-ci-images - images-mirror-missing - - test-dns - validate-selection uses: ./.github/workflows/call-test-deploy.yml with: @@ -192,7 +190,6 @@ jobs: && needs.test.result == 'success' && needs.build-ci-images.result == 'success' && needs.images-mirror-missing.result == 'success' - && needs.test-dns.result == 'success' && (!inputs.chunk_gate || contains(fromJSON('["success","skipped"]'), needs.test-deploy-chunk-0.result)) }} needs: @@ -200,7 +197,7 @@ jobs: - test - build-ci-images - images-mirror-missing - - test-dns + - validate-selection - test-deploy-chunk-0 uses: ./.github/workflows/call-test-deploy.yml with: @@ -224,7 +221,6 @@ jobs: && needs.test.result == 'success' && needs.build-ci-images.result == 'success' && needs.images-mirror-missing.result == 'success' - && needs.test-dns.result == 'success' && (!inputs.chunk_gate || (contains(fromJSON('["success","skipped"]'), needs.test-deploy-chunk-0.result) && contains(fromJSON('["success","skipped"]'), needs.test-deploy-chunk-1.result))) }} @@ -233,7 +229,7 @@ jobs: - test - build-ci-images - images-mirror-missing - - test-dns + - validate-selection - test-deploy-chunk-0 - test-deploy-chunk-1 uses: ./.github/workflows/call-test-deploy.yml @@ -258,7 +254,6 @@ jobs: && needs.test.result == 'success' && needs.build-ci-images.result == 'success' && needs.images-mirror-missing.result == 'success' - && needs.test-dns.result == 'success' && (!inputs.chunk_gate || (contains(fromJSON('["success","skipped"]'), needs.test-deploy-chunk-0.result) && contains(fromJSON('["success","skipped"]'), needs.test-deploy-chunk-1.result) @@ -268,7 +263,7 @@ jobs: - test - build-ci-images - images-mirror-missing - - test-dns + - validate-selection - test-deploy-chunk-0 - test-deploy-chunk-1 - test-deploy-chunk-2 From b7c89f05bd427e2eb0be8066455f9c9c640d9323 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 16 Sep 2026 23:28:02 +0200 Subject: [PATCH 092/197] fix(hlth-csp): let a declared status reach the vhost's onion twin The checker matches accepted codes by hostname, and a clearnet vhost shares only its leftmost label with its onion twin, so --accept-status mirror.=404 never applied to mirror..onion. That vhost serves the same by-design 404, so the probe called it unreachable and systemd failed the service -- with the identical message the missing flag produced, while the status had already moved from 2/INVALIDARGUMENT to 1/FAILURE. --skip-domain carried this derivation all along, in is_skipped_domain; replacing it with --accept-status dropped it. The expansion now runs on the same label rule, in the wrapper rather than the lookup, because the onion address does not exist yet when the unit is rendered -- only the wrapper sees both names, and it reads them from the nginx configs. Co-Authored-By: Claude Opus 5 (1M context) --- roles/sys-ctl-hlth-csp/files/python/script.py | 31 +++++++++- .../test_accept_status_reaches_onions.py | 57 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 tests/unit/python/roles/sys-ctl-hlth-csp/test_accept_status_reaches_onions.py diff --git a/roles/sys-ctl-hlth-csp/files/python/script.py b/roles/sys-ctl-hlth-csp/files/python/script.py index 761ff3159a..7d085e23eb 100755 --- a/roles/sys-ctl-hlth-csp/files/python/script.py +++ b/roles/sys-ctl-hlth-csp/files/python/script.py @@ -57,6 +57,33 @@ def is_skipped_domain(domain: str, skip_set: set[str], skip_labels: set[str]) -> return domain.endswith(".onion") and domain.split(".", 1)[0] in skip_labels +def expand_accept_status(accept_status: list[str], domains: list[str]) -> list[str]: + """ACCEPT_STATUS plus the same codes for every ``.onion`` sibling it covers. + + The checker matches accepted codes by hostname, and a clearnet vhost and + its onion twin share only the leftmost subdomain label, so a code declared + for ``mirror.`` never reaches ``mirror..onion`` โ€” + which serves that same by-design status and would fail the probe. + + Args: + accept_status: ``=[,]`` entries as declared. + domains: every vhost the probe covers, clearnet and onion alike. + """ + onion = [d for d in domains if d.endswith(".onion")] + expanded = list(accept_status) + for entry in accept_status: + host, _, codes = entry.partition("=") + if not codes or host.endswith(".onion"): + continue + label = host.split(".", 1)[0] + expanded.extend( + f"{sibling}={codes}" + for sibling in onion + if sibling.split(".", 1)[0] == label + ) + return expanded + + def detect_scheme_from_conf(conf_path: Path) -> str | None: """ Decide whether this conf listens on HTTP or HTTPS. @@ -310,6 +337,8 @@ def main() -> None: f"--skip-domain: {skipped_present}" ) + accept_status = expand_accept_status(list(args.accept_status or []), domains) + clearnet_domains, onion_domains = split_onion_domains(domains) if onion_domains and not args.tor_proxy: print( @@ -342,7 +371,7 @@ def main() -> None: use_host_network=not bool(args.no_host_network), proxy=batch_proxy, timeout_ms=batch_timeout, - accept_status=list(args.accept_status or []), + accept_status=accept_status, ) rc = rc or batch_rc sys.exit(rc) diff --git a/tests/unit/python/roles/sys-ctl-hlth-csp/test_accept_status_reaches_onions.py b/tests/unit/python/roles/sys-ctl-hlth-csp/test_accept_status_reaches_onions.py new file mode 100644 index 0000000000..770713510c --- /dev/null +++ b/tests/unit/python/roles/sys-ctl-hlth-csp/test_accept_status_reaches_onions.py @@ -0,0 +1,57 @@ +import importlib.util +import unittest + +from . import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "roles" / "sys-ctl-hlth-csp" / "files" / "python" / "script.py" +ONION = "mirror.b5abfs7uwr23x6vbjxqjatyscpmkm6qkmkla7eyapdi4zpwtrz6o4nqd.onion" +DOMAINS = ["mirror.infinito.test", ONION, "html.infinito.test"] + + +def _expand(accept_status, domains=None): + spec = importlib.util.spec_from_file_location("csp_wrapper", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.expand_accept_status( + accept_status, DOMAINS if domains is None else domains + ) + + +class TestAcceptStatusReachesOnions(unittest.TestCase): + def test_the_onion_sibling_inherits_the_declaration(self): + expanded = _expand(["mirror.infinito.test=404"]) + + self.assertIn( + f"{ONION}=404", + expanded, + "the checker matches accepted codes by hostname, so a code declared " + "for the clearnet vhost never reaches its onion twin on its own", + ) + + def test_a_vhost_without_an_onion_twin_gains_nothing(self): + self.assertEqual( + _expand(["html.infinito.test=403"]), ["html.infinito.test=403"] + ) + + def test_several_codes_carry_over_unchanged(self): + expanded = _expand(["mirror.infinito.test=301,302,404"]) + + self.assertIn(f"{ONION}=301,302,404", expanded) + + def test_an_onion_entry_is_not_expanded_again(self): + self.assertEqual(_expand([f"{ONION}=404"]), [f"{ONION}=404"]) + + def test_an_entry_without_codes_is_left_alone(self): + self.assertEqual(_expand(["mirror.infinito.test"]), ["mirror.infinito.test"]) + + def test_only_the_matching_label_is_extended(self): + expanded = _expand(["html.infinito.test=403", "mirror.infinito.test=404"]) + + self.assertEqual( + [e for e in expanded if e.endswith(".onion=404")], [f"{ONION}=404"] + ) + self.assertEqual([e for e in expanded if e.endswith(".onion=403")], []) + + +if __name__ == "__main__": + unittest.main() From 329f43097c7e051423452cba535825cbe89693d2 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 00:31:33 +0200 Subject: [PATCH 093/197] fix(ci): make the image job prove the images exist before it reports success For a fork PR the build step is skipped -- the images come from the privileged run -- so the job went green in 29 seconds while that run was still pushing. Everything gated on it inherited the lie: the DNS test pulled arch 3m30s too early on one run and 53s too early on the next, both times 'manifest unknown', and the four deploy chunks were spared only by starting later. The job now polls until every distro image for its tag is in the registry. It runs unconditionally: where the job built the images itself the first poll passes, and a push that silently failed is caught too. The poll drives scripts/meta/resolve/missing/ci_images.sh, which already owned that check for the release workflow, so the ref shape is not written a fourth time. A resolver that cannot run aborts with 2 rather than reading as 'not yet pushed' and hiding behind the full budget. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/call-images-build-ci.yml | 13 +++++++++ scripts/meta/wait/ci_images.sh | 34 ++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100755 scripts/meta/wait/ci_images.sh diff --git a/.github/workflows/call-images-build-ci.yml b/.github/workflows/call-images-build-ci.yml index 227fd3cfb4..6167740b27 100644 --- a/.github/workflows/call-images-build-ci.yml +++ b/.github/workflows/call-images-build-ci.yml @@ -112,6 +112,7 @@ jobs: run: | set -euo pipefail echo "owner=$(scripts/meta/resolve/repository/owner.sh)" >> "$GITHUB_OUTPUT" + echo "name=$(scripts/meta/resolve/repository/name.sh)" >> "$GITHUB_OUTPUT" - name: ๐Ÿ”จ Build & push every distro (retry 7x, 20s) if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) }} @@ -127,6 +128,18 @@ jobs: RETRY_DELAY_SECONDS: "20" run: ./scripts/image/push_all.sh + - name: โณ Assert every distro image is in the registry + shell: bash + env: + REGISTRY: ghcr.io + OWNER: ${{ steps.owner.outputs.owner }} + REPO_PREFIX: ${{ steps.owner.outputs.name }} + CI_TAG: ${{ steps.out.outputs.image_tag }} + INFINITO_DISTROS: ${{ join(fromJson(steps.out.outputs.matrix), ' ') }} + WAIT_ATTEMPTS: "90" + WAIT_SLEEP_SECONDS: "10" + run: ./scripts/meta/wait/ci_images.sh + - name: ๐Ÿ“ค Upload per-distro build logs if: always() uses: actions/upload-artifact@v7 diff --git a/scripts/meta/wait/ci_images.sh b/scripts/meta/wait/ci_images.sh new file mode 100755 index 0000000000..194c4b081d --- /dev/null +++ b/scripts/meta/wait/ci_images.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Env: +# REGISTRY, OWNER, REPO_PREFIX, CI_TAG, INFINITO_DISTROS +# image coordinates, as scripts/meta/resolve/missing/ci_images.sh reads them +# WAIT_ATTEMPTS polls before giving up (default 60) +# WAIT_SLEEP_SECONDS seconds between polls (default 10) + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +resolver="${script_dir}/../resolve/missing/ci_images.sh" + +wait_attempts="${WAIT_ATTEMPTS:-60}" +wait_sleep_seconds="${WAIT_SLEEP_SECONDS:-10}" + +echo "โณ Waiting for the CI images tagged ${CI_TAG:?}" + +for attempt in $(seq 1 "${wait_attempts}"); do + if ! still_missing="$(bash "${resolver}")"; then + echo "โŒ Could not check the CI images; see the resolver's output above." >&2 + exit 2 + fi + + if [[ "${still_missing}" == "false" ]]; then + echo "โœ… Every CI image is available." + exit 0 + fi + + echo " attempt ${attempt}/${wait_attempts}: still incomplete" + sleep "${wait_sleep_seconds}" +done + +echo "โŒ Gave up after ${wait_attempts} attempts." >&2 +exit 1 From 7623c1680e7c61cea6d6fcea1097e543b6b10da8 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 10:15:48 +0200 Subject: [PATCH 094/197] fix(matrix): pin the MDAD upstream to a commit instead of a branch ref: master was correct while repo pointed at a controlled fork (251ce17c23). When abee277216 moved repo back to the canonical spantaleev upstream it left the branch ref behind, so every deploy took whatever that branch held at that moment. MDAD pins the images it deploys. Its commit cb58394faaef (2026-09-16 20:09Z) raised element-web to v1.12.28, which ships the rewritten room list, and the matrix DM spec went red between two CI runs with no diff of our own. The deployed container carried org.opencontainers.image.version v1.12.28; both the mirror and docker.io still served correct v1.12.27 bytes for the tag this role pins, so neither was the source. 40179e6d4a36 is that branch head at the time of this change, so the checkout is byte-identical to what master would give today and only future drift is removed. MDAD still expects migration_validated_version v2026.05.18.0 at that commit, so its migration gate stays satisfied. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-matrix/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-matrix/meta/services.yml b/roles/web-app-matrix/meta/services.yml index 024e4179be..37d6a76c7a 100644 --- a/roles/web-app-matrix/meta/services.yml +++ b/roles/web-app-matrix/meta/services.yml @@ -131,7 +131,7 @@ matrix: flavor: ansible upstream: repo: https://github.com/spantaleev/matrix-docker-ansible-deploy - ref: master + ref: 40179e6d4a36513eb77c4ef2eff067716ebb1821 migration_validated_version: v2026.05.18.0 runner: image: infinito/web-app-matrix-mdad-runner:1 From 9661800cb09fa25a80e13bb44c5288231fa5652d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 10:18:11 +0200 Subject: [PATCH 095/197] fix(matrix): drive the DM invite through Element's new room list Element 1.12.28 rewrote the room list. Pending invites now sit in their own section that renders collapsed, so no room row exists in the DOM until it is expanded. The predicate iterated an empty list and the poll spent its whole budget with no reachable path to success. Expand every collapsed section first, then match the row by the shape upstream's own test helpers document at revision fe37d199: role=row, aria-level=2, and a button whose aria-label starts with 'Open room'. Of the four previous selectors, three matched nothing in this build and the fourth reached only the Spaces navigation. Match aria-label as well as text, because Element renders the DM partner sometimes as a display name and sometimes as a raw MXID. role=gridcell is deliberately not among them: the cell precedes its own button in document order, so break would land on the DIV while Element's handler sits on the button. The accept regex is unchanged. Element 1.12.28 labels the DM accept 'Start chatting' and still ships 'Accept' and 'Join'; all three are covered and Decline stays excluded. Every click stays a raw DOM click because a persistent device-verification toast overlaps the room list and would intercept an actionability-checked click. Verified on a local compose deploy of variant 0 with sso and tor against 1.12.28: test-dm-admin-biber.js passed in 4.2m, 7 passed, no failures. Co-Authored-By: Claude Opus 5 (1M context) --- .../files/playwright/test-dm-admin-biber.js | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/roles/web-app-matrix/files/playwright/test-dm-admin-biber.js b/roles/web-app-matrix/files/playwright/test-dm-admin-biber.js index d54eda93eb..0ba3371a86 100644 --- a/roles/web-app-matrix/files/playwright/test-dm-admin-biber.js +++ b/roles/web-app-matrix/files/playwright/test-dm-admin-biber.js @@ -97,28 +97,31 @@ exports.register = function (shared) { await adminPage.keyboard.type(bootstrap); await adminPage.keyboard.press("Enter"); - // Biber: wait for admin's invite to propagate, then accept. The flow - // mirrors Element's invite UX: (1) click the sidebar tile for admin's - // invite (the tile renders with admin's display name but no standalone - // Accept button), then (2) click the primary accept action in the invite - // view (modern Element labels this "Start chatting" for DMs; older - // builds / non-DM invites use "Accept" / "Join"). MUST NOT match - // "Decline" / "Decline and block". await expect .poll(async () => { return await biberPage.evaluate(() => { - // Step 1: open the invite. + // Pending invites sit in a section that renders collapsed, so the + // invite row is absent from the DOM until it is expanded. + const sections = document.querySelectorAll( + 'button[aria-expanded="false"][aria-label^="Toggle "]', + ); + for (const section of sections) { + section.click(); + } + const roomTiles = document.querySelectorAll( - "[role='treeitem'], [role='option'], .mx_RoomTile, [data-testid^='room-tile']", + '[role="row"][aria-level="2"] button[aria-label^="Open room"]', ); for (const tile of roomTiles) { - const text = (tile.textContent || "").trim(); - if (/administrator/i.test(text)) { + const name = `${tile.getAttribute("aria-label") || ""} ${tile.textContent || ""}`; + if (/administrator/i.test(name)) { tile.click(); break; } } - // Step 2: click accept. + + // Anchored: "Decline and block" contains an accept-looking word and + // clicking it destroys the invite. const acceptCandidates = document.querySelectorAll( "button, a, [role='button']", ); @@ -128,9 +131,6 @@ exports.register = function (shared) { b.click(); } } - // Success: invite accepted when the room timeline renders a - // message composer for biber (only appears once membership is - // `join`). Use the same composer signature as admin's side. return !!document.querySelector( "div[role='textbox'][contenteditable='true'], textarea[aria-label*='message' i]", ); From df4b7757bc9f0dc6985406dade439d6e0205ad85 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 11:49:50 +0200 Subject: [PATCH 096/197] fix(hlth-csp): take the csp-checker build that stops failing on a 304 csp-checker 3.0.0 narrowed the healthy set to 2xx by failing every status at or above 300, which swept in 304. The health service then died on the first revalidated vhost: in CI job 104991065851 it reported element. unreachable with Status 304, and sys-service-terminator retried the unit 25 times over 1792 seconds before the job failed. 3.0.1 takes 304 out of the failure branch. The checker's own e2e suite verifies it in both directions: with the fix the second visit to a revalidating fixture reports 304 and stays healthy, without it the fixture reproduces the CI line verbatim. Co-Authored-By: Claude Opus 5 (1M context) --- roles/sys-ctl-hlth-csp/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sys-ctl-hlth-csp/meta/services.yml b/roles/sys-ctl-hlth-csp/meta/services.yml index bb5a8b194a..90ffed8219 100644 --- a/roles/sys-ctl-hlth-csp/meta/services.yml +++ b/roles/sys-ctl-hlth-csp/meta/services.yml @@ -8,4 +8,4 @@ csp: categories: [web] csp-checker: image: ghcr.io/kevinveenbirkenbach/csp-checker - version: 3.0.0 + version: 3.0.1 From ada2e4d3119c88ada7b863b6b8dce45d126920c9 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 12:21:41 +0200 Subject: [PATCH 097/197] fix(mirror): keep the origin prefix when an upstream redirects The mirror proxies // to https:/// and relayed the upstream Location header untouched. A root-relative Location then resolved against the mirror instead of the origin, the prefix that names the origin was lost, the follow-up request matched no location block and fell into the catch-all 404. Chromium discards a 404 served for a script request, so the page lost the asset with net::ERR_BLOCKED_BY_ORB. MIG job 105068912392 shows both halves, 28 times each: GET /unpkg.com/3d-force-graph/dist/3d-force-graph.min.js answered 302, then GET /3d-force-graph@1.80.0/dist/3d-force-graph.min.js answered 404. unpkg redirects an unversioned path to the resolved version and expresses the target as an absolute path. The forms move into a redirect_sources filter so the template states the intent once and the list is unit-tested. Measured against a two-container nginx fixture: an absolute https Location, an absolute http one and a root-relative one all come back as //target. A Location naming a third host stays untouched, which is unchanged behaviour and still leaks past the mirror. A network-path form was written first and then dropped: nginx normalises such a target in return, and routing one through add_header emits two Location headers of which the proxy relays only the first, so the case could not be produced and the rule stayed unproven. Co-Authored-By: Claude Opus 5 (1M context) --- plugins/filter/redirect_sources.py | 28 ++++++++ roles/web-svc-mirror/templates/proxy.conf.j2 | 3 + .../plugins/filter/test_redirect_sources.py | 72 +++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 plugins/filter/redirect_sources.py create mode 100644 tests/unit/python/plugins/filter/test_redirect_sources.py diff --git a/plugins/filter/redirect_sources.py b/plugins/filter/redirect_sources.py new file mode 100644 index 0000000000..f826647bb7 --- /dev/null +++ b/plugins/filter/redirect_sources.py @@ -0,0 +1,28 @@ +from ansible.errors import AnsibleFilterError + + +def redirect_sources(host: str) -> list[str]: + """Ordered match prefixes for a redirect that targets ``host``. + + Args: + host: the upstream host, as it appears in the proxied URL. + + Returns: + The two host-qualified forms, then the bare ``/`` for a target the + upstream expressed as an absolute path. + """ + if not isinstance(host, str) or not host.strip(): + raise AnsibleFilterError("redirect_sources: host must be a non-empty string") + stripped = host.strip() + return [ + f"https://{stripped}/", + f"http://{stripped}/", + "/", + ] + + +class FilterModule: + def filters(self): + return { + "redirect_sources": redirect_sources, + } diff --git a/roles/web-svc-mirror/templates/proxy.conf.j2 b/roles/web-svc-mirror/templates/proxy.conf.j2 index 4dc8bd6c80..0f2016d735 100644 --- a/roles/web-svc-mirror/templates/proxy.conf.j2 +++ b/roles/web-svc-mirror/templates/proxy.conf.j2 @@ -22,6 +22,9 @@ server proxy_ssl_server_name on; proxy_ssl_name {{ origin_host }}; proxy_set_header Host {{ origin_host }}; +{% for redirect_source in origin_host | redirect_sources %} + proxy_redirect {{ redirect_source }} /{{ origin_host }}/; +{% endfor %} proxy_cache cache; proxy_cache_valid 200 301 302 168h; proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; diff --git a/tests/unit/python/plugins/filter/test_redirect_sources.py b/tests/unit/python/plugins/filter/test_redirect_sources.py new file mode 100644 index 0000000000..86149503b1 --- /dev/null +++ b/tests/unit/python/plugins/filter/test_redirect_sources.py @@ -0,0 +1,72 @@ +import sys +import unittest + +from . import PROJECT_ROOT + +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from ansible.errors import AnsibleFilterError + +from plugins.filter.redirect_sources import FilterModule, redirect_sources + + +class TestRedirectSources(unittest.TestCase): + def test_every_reference_form_is_covered(self): + self.assertEqual( + redirect_sources("unpkg.com"), + [ + "https://unpkg.com/", + "http://unpkg.com/", + "/", + ], + ) + + def test_the_catch_all_comes_last(self): + sources = redirect_sources("unpkg.com") + + self.assertEqual( + sources[-1], + "/", + "the bare slash is the widest form and reads as the fallback only " + "while it stays at the end", + ) + + def test_no_form_is_a_prefix_of_another(self): + sources = redirect_sources("unpkg.com") + + for index, source in enumerate(sources): + for other in sources[:index]: + self.assertFalse( + source.startswith(other), + f"{source!r} starts with {other!r}, so the two forms " + f"cannot be told apart by a prefix comparison", + ) + + def test_surrounding_whitespace_is_dropped(self): + self.assertEqual(redirect_sources(" unpkg.com ")[0], "https://unpkg.com/") + + def test_the_host_is_not_otherwise_rewritten(self): + self.assertEqual(redirect_sources("CDN.Example.COM")[1], "http://CDN.Example.COM/") + + def test_a_subdomain_host_keeps_every_label(self): + self.assertEqual( + redirect_sources("cdn.jsdelivr.net")[0], "https://cdn.jsdelivr.net/" + ) + + def test_an_empty_host_is_rejected(self): + for value in ("", " "): + with self.assertRaises(AnsibleFilterError): + redirect_sources(value) + + def test_a_non_string_host_is_rejected(self): + for value in (None, 42, ["unpkg.com"]): + with self.assertRaises(AnsibleFilterError): + redirect_sources(value) + + def test_the_filter_is_exported(self): + self.assertIs(FilterModule().filters()["redirect_sources"], redirect_sources) + + +if __name__ == "__main__": + unittest.main() From ed7a012fafeea6aa5b63fa6d783bede5c2c237eb Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 12:23:02 +0200 Subject: [PATCH 098/197] fix(socialhome): declare the font origins the app itself loads socialhome's own page head links a Google Fonts stylesheet. The deployment CSP did not know the origin, so the browser blocked it, csp-checker reported the violation, the hlth-csp unit failed and sys-service-terminator took the job down with it. Job 104991069038 carries the only violation in the whole bundle, 52 times: [CSP DOM] Blocked URI: https://fonts.googleapis.com/css2?family=Fraunces:... with Directive: style-src-elem. Fraunces appears nowhere in this repository, so the reference comes from upstream socialhome rather than from our theming. font-src for fonts.gstatic.com rides along: the stylesheet pulls the font files from there, so declaring only the stylesheet origin would have surfaced the next violation instead of clearing the check. That half is derived from how @font-face resolves and from the four sibling roles that declare the same pair, not observed, because the stylesheet never loaded. This keeps a clearnet origin reachable from an onion deployment, which the mirror exists to avoid. The sibling roles carry a standing TODO to move such assets to the mirror; socialhome joins that debt rather than adding a new kind of it. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-socialhome/meta/csp.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/web-app-socialhome/meta/csp.yml b/roles/web-app-socialhome/meta/csp.yml index 5d7ccc87ce..6258ff8fbc 100644 --- a/roles/web-app-socialhome/meta/csp.yml +++ b/roles/web-app-socialhome/meta/csp.yml @@ -4,3 +4,7 @@ whitelist: - "blob:" script-src: - "blob:" + style-src-elem: + - https://fonts.googleapis.com # nocheck: external-cdn,url + font-src: + - https://fonts.gstatic.com # nocheck: external-cdn,url From d9cc2e2833324f4c1af4432e74c6e1bae2f7b0e9 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 12:35:00 +0200 Subject: [PATCH 099/197] test(swarm): make a dnsmasq that stops answering describable Penpot job 105068912535 and MediaWiki job 105068912565 both died on one lab node whose dnsmasq kept its :53 socket bound and rejected every query with 'ignoring query from non-local network', once for a container address and once for 127.0.0.1. Rejecting loopback as non-local only makes sense if --local-service is active and dnsmasq's own interface list went stale, but the bundle records neither the flag nor where it would come from. Capture the process command line, /etc/default/dnsmasq, the unit including its drop-ins, and the config that actually landed on disk. Addresses and listening sockets were already collected and are not duplicated. Read-only. Every call falls back to a text line when the file or the unit is absent, so a node without dnsmasq reports that rather than failing the collector. Co-Authored-By: Claude Opus 5 (1M context) --- .../deploy/swarm/utils/collect/diagnostics.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/tests/deploy/swarm/utils/collect/diagnostics.sh b/scripts/tests/deploy/swarm/utils/collect/diagnostics.sh index 3684745315..6fa5376a3e 100755 --- a/scripts/tests/deploy/swarm/utils/collect/diagnostics.sh +++ b/scripts/tests/deploy/swarm/utils/collect/diagnostics.sh @@ -144,6 +144,20 @@ for node in "${MGR}" "${WRK1}" "${WRK2}" "${NFS_SERVER}"; do dexec "${node}" sh -c "ss -lunp 2>/dev/null; ss -lntp 2>/dev/null" || echo "(ss unavailable)" echo "--- dnsmasq journal ---" dexec "${node}" sh -c "journalctl -u dnsmasq --no-pager 2>&1" || echo "(journalctl unavailable)" + echo "--- dnsmasq command line ---" + # shellcheck disable=SC2016 + dexec "${node}" sh -c 'pid=$(systemctl show -p MainPID --value dnsmasq 2>/dev/null) +if [ -n "${pid}" ] && [ "${pid}" != "0" ] && [ -r "/proc/${pid}/cmdline" ]; then + tr "\000" "\n" < "/proc/${pid}/cmdline" +else + echo "(dnsmasq has no readable MainPID)" +fi' || echo "(dnsmasq cmdline unavailable)" + echo "--- dnsmasq startup wiring ---" + dexec "${node}" sh -c 'cat /etc/default/dnsmasq 2>&1 || echo "(no /etc/default/dnsmasq)"' + dexec "${node}" sh -c "systemctl cat dnsmasq 2>&1" || echo "(systemctl cat unavailable)" + echo "--- dnsmasq config on disk ---" + dexec "${node}" sh -c 'cat /etc/dnsmasq.conf 2>&1 || echo "(no /etc/dnsmasq.conf)"' + dexec "${node}" sh -c 'ls -l /etc/dnsmasq.d 2>&1 || echo "(no /etc/dnsmasq.d)"' echo "--- addresses ---" dexec "${node}" ip -4 addr show echo "--- nat rules ---" From f3b624e2d9923f69591afb2f659acda838c747c6 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 13:12:24 +0200 Subject: [PATCH 100/197] fix(swarm-lab): close the second consumer of the broken interface list 5aa7506e36 traced the enumeration bug: a veth that disappears between the RTM_GETADDR snapshot and the SIOCGIFFLAGS ioctl makes the AF_INET6 pass return 0, and network.c:836 then skips the AF_INET pass entirely. That commit removed bind-dynamic, which took the listener collector out of the blast radius. It left the other consumer of daemon->interface_addrs in place. forward.c:1669 opens a rejection gate under OPT_LOCAL_SERVICE and walks the same list. With the IPv4 half missing, every IPv4 source is rejected, loopback included, and the once-only warning at forward.c:1695 makes a permanent outage look like a single incident. Penpot job 105068912535 and MediaWiki job 105068912565 each lost one lab node this way: PID 332 ran with --local-service and is the PID that logged the rejection; the sampler shows 179 consecutive failures over 47 minutes with the socket still bound. Debian's init-system-common appends --local-service unconditionally, so no packaging knob removes it. option.c:6240 resets it when any of if_names, if_except, if_addrs or authserver is set, no matter which config file carried the directive. interface=* fills if_names and, through wildcard_match, still admits every interface including a docker_gwbridge that does not exist at render time. bind-interfaces and bind-dynamic are absent from that predicate and would not have helped. Measured against the rendered template with --local-service on argv as Debian appends it: the DNS service limited to local subnets line is gone and dig still answers from the lab zone. The nodes run Debian and Ubuntu, so the task name's previous claim about fedora and centos, and about local-service implying bind-interfaces, was wrong twice over and is replaced. Co-Authored-By: Claude Opus 5 (1M context) --- compose/swarm/playbook.yml | 2 +- compose/swarm/templates/dnsmasq.conf.j2 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/swarm/playbook.yml b/compose/swarm/playbook.yml index a4e11016c3..6be12b1fb1 100644 --- a/compose/swarm/playbook.yml +++ b/compose/swarm/playbook.yml @@ -134,7 +134,7 @@ timeout: 30 when: lab_is_mgr or inventory_hostname in [lab_nfs, lab_backup] - - name: own the whole dnsmasq config (fedora and centos ship local-service, which implies bind-interfaces and dnsmasq refuses next to our bind-dynamic) + - name: own the whole dnsmasq config (the packaging appends --local-service unconditionally and only an interface, except-interface, listen-address or auth-server directive resets it; without that reset a failed interface re-enumeration empties dnsmasq's local-subnet list and it rejects every query, loopback included, until an address event rebuilds it) template: # nocheck: stack-host-template - lab node system config, every DNS node writes its own /etc/dnsmasq.conf and IS_STACK_HOST does not exist in this play src: dnsmasq.conf.j2 dest: /etc/dnsmasq.conf diff --git a/compose/swarm/templates/dnsmasq.conf.j2 b/compose/swarm/templates/dnsmasq.conf.j2 index ef198a34b5..48dfe761fb 100644 --- a/compose/swarm/templates/dnsmasq.conf.j2 +++ b/compose/swarm/templates/dnsmasq.conf.j2 @@ -5,3 +5,4 @@ server={{ resolver }} {% endfor %} address=/{{ lab_domain }}/{{ lab_domain_target }} local=/{{ lab_domain }}/ +interface=* From be6a25517ce7e7295996c63f881fae4e0bad34dc Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 16:39:16 +0200 Subject: [PATCH 101/197] feat(docker): classify image pins and build pull references from the class A digest pin was emitted as `name:sha256:...`, which no registry accepts: image_source('ghcr.io/.../csp-checker', 'sha256:d559...') returned 'ghcr.io/.../csp-checker:3.0.0'. The separator depends on the pin class, so the class has to be decided before the reference is built. utils/docker/image/pin.py names the three classes in preference order (semver, then digest, then a general ref that no registry can order) and maps each to the nocheck marker that declares it. Both discovery entry points now end in pull_reference(), so '@' versus ':' is decided in one place instead of at every call site. The semver pin lint points a digest at 'docker-digest' instead of 'docker-version', so the two fallbacks stay distinguishable in the suppression catalog. It stays warn-only. Co-Authored-By: Claude Opus 5 (1M context) --- .../update/docker/test_image_reachable.py | 13 ++- .../ansible/services/test_semver_pinning.py | 12 +-- .../python/utils/docker/test_image_pin.py | 88 +++++++++++++++++++ utils/docker/image/discovery.py | 5 +- utils/docker/image/pin.py | 59 +++++++++++++ 5 files changed, 166 insertions(+), 11 deletions(-) create mode 100644 tests/unit/python/utils/docker/test_image_pin.py create mode 100644 utils/docker/image/pin.py diff --git a/tests/external/update/docker/test_image_reachable.py b/tests/external/update/docker/test_image_reachable.py index 9065c44535..362d1e5ec9 100644 --- a/tests/external/update/docker/test_image_reachable.py +++ b/tests/external/update/docker/test_image_reachable.py @@ -17,6 +17,7 @@ import unittest from utils.annotations.message import warning from utils.docker.image.discovery import iter_role_images, load_yaml +from utils.docker.image.pin import pull_reference from utils.docker.registry import manifest_exists from utils.roles.mapping import ROLE_FILE_META_SERVICES from utils.update.base import resolve_max_fetch_workers @@ -28,6 +29,10 @@ def _pull_image(ref) -> str: return ref.name if ref.registry == "docker.io" else f"{ref.registry}/{ref.name}" +def _display(ref) -> str: + return pull_reference(_pull_image(ref), ref.version) + + class TestDockerImageReachable(unittest.TestCase): """Fail on any pinned Docker image:version tag the registry reports absent.""" @@ -57,7 +62,7 @@ class TestDockerImageReachable(unittest.TestCase): for ref in indeterminate: warning( - f"{ref.role}/{ref.service}: {_pull_image(ref)}:{ref.version} " + f"{ref.role}/{ref.service}: {_display(ref)} " f"could not be verified (network / auth / rate-limit)", title="๐Ÿ” Unverified Docker image", file=f"roles/{ref.role}/{ROLE_FILE_META_SERVICES}", @@ -65,12 +70,12 @@ class TestDockerImageReachable(unittest.TestCase): if missing: lines = "\n".join( - f" {ref.role}/{ref.service}: {_pull_image(ref)}:{ref.version}" + f" {ref.role}/{ref.service}: {_display(ref)}" for ref in sorted(missing, key=lambda r: (r.role, r.service)) ) self.fail( - "These pinned Docker image:version tags do not exist in their " - f"registry (HTTP 404):\n{lines}" + "These pinned Docker images do not exist in their registry " + f"(HTTP 404):\n{lines}" ) def test_incomplete_image_version_pairs_warn(self) -> None: diff --git a/tests/lint/ansible/services/test_semver_pinning.py b/tests/lint/ansible/services/test_semver_pinning.py index 07e1df230e..c09544eda9 100644 --- a/tests/lint/ansible/services/test_semver_pinning.py +++ b/tests/lint/ansible/services/test_semver_pinning.py @@ -59,6 +59,7 @@ from utils.annotations.message import warning as gha_warning from utils.annotations.suppress import is_suppressed_at from utils.cache.files import read_text from utils.cache.yaml import load_yaml_any +from utils.docker.image.pin import is_digest from utils.roles.mapping import ROLE_FILE_META_SERVICES from utils.update.base import is_semver @@ -69,6 +70,7 @@ if TYPE_CHECKING: _DOCKER_RULE = "docker-version" +_DIGEST_RULE = "docker-digest" _REPO_RULE = "repository-version" _DOCKER_TITLE = "Non-semver Docker image version" @@ -176,7 +178,7 @@ class TestSemverPinning(unittest.TestCase): line_no = _locate_value_line( lines, "version", value, used_version_lines ) - rule = _DOCKER_RULE + rule = _DIGEST_RULE if is_digest(value) else _DOCKER_RULE title = _DOCKER_TITLE category: type[Warning] = NonSemverImageVersionWarning key_label = "version" @@ -208,10 +210,10 @@ class TestSemverPinning(unittest.TestCase): if findings_emitted: print( - "\n๐Ÿ’ก Non-semver pins: suppress with " - "'# nocheck: docker-version' (above or on the `version:` line) " - "or '# nocheck: repository-version' (above or on the `ref:` " - "line) when the non-semver pin is intentional.", + "\n๐Ÿ’ก Non-semver pins: a digest is the preferred fallback and " + "declares itself with '# nocheck: docker-digest'; a general " + "tag uses '# nocheck: docker-version' and a general git ref " + "'# nocheck: repository-version', above or on the pinned line.", flush=True, ) diff --git a/tests/unit/python/utils/docker/test_image_pin.py b/tests/unit/python/utils/docker/test_image_pin.py new file mode 100644 index 0000000000..1b7b7e949f --- /dev/null +++ b/tests/unit/python/utils/docker/test_image_pin.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import unittest + +from utils.docker.image.discovery import docker_hub_source, image_source +from utils.docker.image.pin import ( + DIGEST, + REF, + RULE_BY_CLASS, + SEMVER, + is_digest, + pin_class, + pull_reference, + reference_separator, +) + +DIGEST_VALUE = "sha256:d5591131b1d898cd02fc7a17adb8ea026352d67809bc9289707778e9e03a76a4" + + +class TestPinClass(unittest.TestCase): + def test_a_semver_tag_is_a_semver_pin(self): + for value in ("3.0.1", "16", "17-3.5", "5.4.5-php8.3-apache", "v1.12.27"): + self.assertEqual(pin_class(value), SEMVER, value) + + def test_a_content_address_is_a_digest_pin(self): + self.assertEqual(pin_class(DIGEST_VALUE), DIGEST) + + def test_a_moving_name_is_neither(self): + for value in ("latest", "main", "stable", "buildx-stable-1"): + self.assertEqual(pin_class(value), REF, value) + + def test_surrounding_whitespace_does_not_change_the_class(self): + self.assertEqual(pin_class(f" {DIGEST_VALUE} "), DIGEST) + self.assertEqual(pin_class(" 3.0.1 "), SEMVER) + + def test_every_class_has_its_own_rule(self): + rules = [RULE_BY_CLASS[SEMVER], RULE_BY_CLASS[DIGEST], RULE_BY_CLASS[REF]] + + self.assertEqual(len(set(rules)), 3, "a shared rule cannot express intent") + self.assertEqual(rules, ["docker-version", "docker-digest", "docker-ref"]) + + +class TestPullReference(unittest.TestCase): + def test_a_digest_is_joined_with_an_at_sign(self): + self.assertEqual(reference_separator(DIGEST_VALUE), "@") + self.assertEqual( + pull_reference("ghcr.io/owner/app", DIGEST_VALUE), + f"ghcr.io/owner/app@{DIGEST_VALUE}", + ) + + def test_a_tag_is_joined_with_a_colon(self): + self.assertEqual(reference_separator("3.0.1"), ":") + self.assertEqual(pull_reference("ghcr.io/owner/app", "3.0.1"), "ghcr.io/owner/app:3.0.1") + + def test_is_digest_rejects_a_tag_that_merely_mentions_sha256(self): + self.assertFalse(is_digest("v2-sha256:abc")) + + +class TestSourceKeepsTheDigest(unittest.TestCase): + """A pull ref that silently drops the digest pins nothing at all.""" + + def test_image_source_keeps_a_digest(self): + self.assertEqual( + image_source("ghcr.io/owner/app", DIGEST_VALUE), + f"ghcr.io/owner/app@{DIGEST_VALUE}", + ) + + def test_image_source_still_tags_a_semver(self): + self.assertEqual(image_source("ghcr.io/owner/app", "3.0.1"), "ghcr.io/owner/app:3.0.1") + + def test_docker_hub_source_keeps_a_digest(self): + self.assertEqual( + docker_hub_source("postgres", DIGEST_VALUE), + f"docker.io/library/postgres@{DIGEST_VALUE}", + ) + + def test_a_digest_never_yields_a_tag_separator(self): + source = image_source("quay.io/owner/app", DIGEST_VALUE) + + self.assertNotIn( + f":{DIGEST_VALUE}", + source, + "a colon here would make the registry read the digest as a tag name", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/docker/image/discovery.py b/utils/docker/image/discovery.py index c569aed7c1..559003e0e7 100644 --- a/utils/docker/image/discovery.py +++ b/utils/docker/image/discovery.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING from utils.cache.yaml import load_yaml_any as _load_yaml_cached +from utils.docker.image.pin import pull_reference from utils.roles.mapping import ROLE_FILE_META_SERVICES if TYPE_CHECKING: @@ -146,7 +147,7 @@ def docker_hub_source(image: str, version: str) -> str: if "/" not in base: base = f"library/{base}" - return f"docker.io/{base}:{version}" + return pull_reference(f"docker.io/{base}", version) def image_source(image: str, version: str) -> str: @@ -166,7 +167,7 @@ def image_source(image: str, version: str) -> str: if registry == "docker.io" and "/" not in base: base = f"library/{base}" - return f"{registry}/{base}:{version}" + return pull_reference(f"{registry}/{base}", version) def canonical_image_name(image: str) -> str: diff --git a/utils/docker/image/pin.py b/utils/docker/image/pin.py new file mode 100644 index 0000000000..4ddfce88e5 --- /dev/null +++ b/utils/docker/image/pin.py @@ -0,0 +1,59 @@ +from utils.update.base import version_depth + +DIGEST_PREFIX = "sha256:" + +SEMVER = "semver" +DIGEST = "digest" +REF = "ref" + +RULE_BY_CLASS = { + SEMVER: "docker-version", + DIGEST: "docker-digest", + REF: "docker-ref", +} + + +def is_digest(version: str) -> bool: + """Whether ``version`` pins by content rather than by name. + + Args: + version: the ``version:`` value a role declares for an image. + """ + return str(version).strip().startswith(DIGEST_PREFIX) + + +def pin_class(version: str) -> str: + """Classify a pin as ``semver``, ``digest`` or ``ref``. + + Args: + version: the ``version:`` value a role declares for an image. + + Returns: + The class name. ``ref`` is the residue: a tag that names no version + the registry can order, so nothing can tell whether it moved. + """ + stripped = str(version).strip() + if is_digest(stripped): + return DIGEST + if version_depth(stripped) > 0: + return SEMVER + return REF + + +def reference_separator(version: str) -> str: + """The character that joins an image name to ``version`` in a pull ref. + + Args: + version: the ``version:`` value a role declares for an image. + """ + return "@" if is_digest(version) else ":" + + +def pull_reference(name: str, version: str) -> str: + """Join an image name and a pin into the reference a registry accepts. + + Args: + name: image name, with or without a registry prefix. + version: the ``version:`` value a role declares for an image. + """ + return f"{name}{reference_separator(version)}{str(version).strip()}" From 51dc06ad098b6e8e73f43c5490c8e2b2932b6fb0 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 16:44:28 +0200 Subject: [PATCH 102/197] feat(update): order vendor patch counters and five-component versions Two tag shapes in this repository name a fixed release yet failed the semver gate, so the bump job skipped them and nothing could tell whether a newer one existed: the Checkmk counter '2.4.0p32' and the Collabora five-component '26.04.2.1.1'. Both returned depth 0 and fell into the unorderable residue next to 'latest' and 'alpine'. The numeric core now takes up to five components, and a counter written as a letter plus a number is parsed into its own group. The letter joins the flavor and the number joins the ordering key, which is what keeps '2.4.0p32' in a different family from the four-component '2.4.0.32': a bump between them would silently change what the role deploys. The number in the key also makes p9 order below p10 instead of above it. version_flavor reads the parse result positionally, so the wider tuple leaves it untouched. Its contract does change for counter tags, which now report the letter as their flavor; every shape the grammar already handled keeps its classification, pinned by a regression case in the new test. Co-Authored-By: Claude Opus 5 (1M context) --- .../utils/update/test_base_pin_families.py | 101 ++++++++++++++++++ utils/update/base.py | 40 ++++--- 2 files changed, 125 insertions(+), 16 deletions(-) create mode 100644 tests/unit/python/utils/update/test_base_pin_families.py diff --git a/tests/unit/python/utils/update/test_base_pin_families.py b/tests/unit/python/utils/update/test_base_pin_families.py new file mode 100644 index 0000000000..fe53566fd6 --- /dev/null +++ b/tests/unit/python/utils/update/test_base_pin_families.py @@ -0,0 +1,101 @@ +"""Immutable tags the updater could not order before. + +Two shapes reach the repository that name a fixed release yet failed the semver +gate, so the bump job skipped them and nothing could tell whether a newer one +existed: a vendor patch counter such as ``2.4.0p32`` and a five-component +version such as ``26.04.2.1.1``. +""" + +from __future__ import annotations + +import unittest + +from utils.update.base import ( + is_semver, + latest_semver, + version_depth, + version_flavor, + version_key, +) + + +def _latest(current: str, tags: list[str]) -> str | None: + return latest_semver(tags, version_depth(current), version_flavor(current)) + + +class TestPatchCounterTags(unittest.TestCase): + def test_a_patch_counter_is_orderable(self): + self.assertTrue(is_semver("2.4.0p32")) + + def test_it_bumps_within_its_own_counter(self): + self.assertEqual(_latest("2.4.0p32", ["2.4.0p32", "2.4.0p33"]), "2.4.0p33") + + def test_the_counter_orders_numerically_not_lexically(self): + self.assertEqual(_latest("2.4.0p9", ["2.4.0p9", "2.4.0p10"]), "2.4.0p10") + + def test_a_later_release_wins_over_a_later_counter(self): + self.assertEqual(_latest("2.4.0p32", ["2.4.0p99", "2.4.1p1"]), "2.4.1p1") + + def test_it_never_crosses_into_a_four_component_version(self): + self.assertEqual( + _latest("2.4.0p32", ["2.4.0p32", "2.4.0.99"]), + "2.4.0p32", + "2.4.0.99 is a different tag family and bumping into it would " + "silently change what the role deploys", + ) + + def test_the_counter_letter_keeps_families_apart(self): + self.assertEqual(version_flavor("2.4.0p32"), "p") + self.assertEqual(version_flavor("2.4.0.32"), "") + self.assertNotEqual(version_depth("2.4.0p32"), version_depth("2.4.0.32")) + + def test_a_plain_release_does_not_bump_into_a_counter(self): + self.assertEqual(_latest("2.4.0", ["2.4.0", "2.4.0p1"]), "2.4.0") + + +class TestFiveComponentTags(unittest.TestCase): + def test_five_components_are_orderable(self): + self.assertTrue(is_semver("26.04.2.1.1")) + self.assertEqual(version_depth("26.04.2.1.1"), 5) + + def test_it_bumps_within_its_own_depth(self): + self.assertEqual( + _latest("26.04.2.1.1", ["26.04.2.1.1", "26.04.2.1.2"]), "26.04.2.1.2" + ) + + def test_it_never_crosses_into_a_shorter_version(self): + self.assertEqual( + _latest("26.04.2.1.1", ["26.04.2.1.1", "27.0.0.0"]), "26.04.2.1.1" + ) + + +class TestNothingElseChanged(unittest.TestCase): + """The shapes the updater already handled must keep their family key.""" + + def test_known_shapes_keep_their_classification(self): + cases = { + "1.2.3": (True, 3, "", (1, 2, 3, 0)), + "v1.12.27": (True, 3, "", (1, 12, 27, 0)), + "16": (True, 1, "", (16, 0, 0, 0)), + "5.4.5-php8.3-apache": (True, 3, "-php8.3-apache", (5, 4, 5, 0)), + } + + for tag, (semver, depth, flavor, key) in cases.items(): + self.assertEqual(is_semver(tag), semver, tag) + self.assertEqual(version_depth(tag), depth, tag) + self.assertEqual(version_flavor(tag), flavor, tag) + self.assertEqual(version_key(tag), key, tag) + + def test_moving_names_are_still_refused(self): + for tag in ("latest", "stable", "lts", "alpine", "main"): + self.assertFalse(is_semver(tag), tag) + + def test_a_flavored_tag_still_refuses_a_different_runtime(self): + self.assertEqual( + _latest("5.4.5-php8.3-apache", ["5.4.6-php8.4-apache", "5.4.6"]), + None, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/update/base.py b/utils/update/base.py index 2900d2eaa7..bd10ac796f 100644 --- a/utils/update/base.py +++ b/utils/update/base.py @@ -1,15 +1,18 @@ """Semver primitives shared by every version-bump backend. -A "version" here is a tag of the shape ``v?(-)?``, -where the numeric part has 1 to 4 dot-separated components and the -optional ``-`` suffix is an opaque discriminator (e.g. the -Docker Official Image tag ``5.4.5-php8.3-apache`` or the npm-style -``1.2.3-alpha``). +A "version" here is a tag of the shape +``v??(-)?``, where the numeric part has 1 to 5 +dot-separated components, the optional ```` is a vendor counter written +as a letter and a number (e.g. the Checkmk tag ``2.4.0p32``), and the optional +``-`` suffix is an opaque discriminator (e.g. the Docker Official Image +tag ``5.4.5-php8.3-apache`` or the npm-style ``1.2.3-alpha``). -Upgrade candidates MUST share the same depth (1, 2, 3 or 4 components) -AND the same flavor as the current tag, so that ``5.4.5-php8.3-apache`` -never silently bumps to ``5.4.6-php8.4-apache`` (different runtime) or -to ``5.4.6`` (different depth). +Upgrade candidates MUST share the same depth (1 to 5 components) AND the same +flavor as the current tag, so that ``5.4.5-php8.3-apache`` never silently bumps +to ``5.4.6-php8.4-apache`` (different runtime) or to ``5.4.6`` (different +depth). The patch letter joins the flavor and its number joins the ordering +key, which is what keeps ``2.4.0p32`` ordering against ``2.4.0p33`` and never +against the four-component ``2.4.0.32``. """ from __future__ import annotations @@ -17,15 +20,19 @@ from __future__ import annotations import os import re -_SEMVER_CORE = r"v?\d+(?:\.\d+){0,3}" -_VERSIONED_TAG_RE = re.compile(rf"^(?P{_SEMVER_CORE})(?P-\S+)?$") +_SEMVER_CORE = r"v?\d+(?:\.\d+){0,4}" +_VERSIONED_TAG_RE = re.compile( + rf"^(?P{_SEMVER_CORE})(?P[A-Za-z]\d+)?(?P-\S+)?$" +) -def _parse_versioned_tag(tag: str) -> tuple[str, str] | None: +def _parse_versioned_tag(tag: str) -> tuple[str, str, int | None] | None: match = _VERSIONED_TAG_RE.match(str(tag).strip()) if match is None: return None - return match.group("semver"), match.group("flavor") or "" + patch = match.group("patch") + flavor = (match.group("flavor") or "") + (patch[0] if patch else "") + return match.group("semver"), flavor, int(patch[1:]) if patch else None def is_semver(value: str) -> bool: @@ -36,16 +43,17 @@ def version_key(tag: str) -> tuple[int, ...]: parsed = _parse_versioned_tag(tag) if parsed is None: return (0,) * 4 - semver, _flavor = parsed + semver, _flavor, patch = parsed parts = tuple(int(part) for part in semver.lstrip("v").split(".")) - return parts + (0,) * (4 - len(parts)) + padded = parts + (0,) * (4 - len(parts)) + return padded if patch is None else (*padded, patch) def version_depth(tag: str) -> int: parsed = _parse_versioned_tag(tag) if parsed is None: return 0 - semver, _flavor = parsed + semver, _flavor, _patch = parsed return len(semver.lstrip("v").split(".")) From da665ab699524dc49fe00f44225102d42824bcea Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 16:58:40 +0200 Subject: [PATCH 103/197] feat(services): give fifteen moving pins a name the registry can order A pin like 'alpine', 'stable', 'lts' or 'buildx-stable-1' names no version, so nothing can compare it to anything: the bump job skips it and the pin drifts with whatever upstream last pushed under that name. Each of these fifteen now names its release, which is the precondition for the freshness pipeline to see it at all. Every conversion was placed against the tag it replaces, and none of them is a downgrade. Six resolve to a byte-identical digest and are pure relabelling (memcached, redis, the two debian bases, jenkins, xwiki); four are genuinely newer than what the moving tag carried, so those roles will deploy a later version than before (nginx moves past the 1.31.5 behind 'alpine', buildkit past the v0.32.2 behind 'buildx-stable-1', dockerize past a 'latest' older than v0.10.0); openresty and pretix keep their version and differ only by build. fider is the one that loses something: 'stable' carries a rebuild pushed twelve days after v0.36.0, which is still the newest versioned tag upstream publishes. An unnamed rebuild cannot be tracked, so the pin takes the named release and the updater can raise it when v0.37.0 lands. The flavor invariant keeps the families apart: jenkins stays on '-lts' and xwiki inside '-postgres-tomcat', so no bump can cross into a different runtime. svc-runner loses a 'nocheck: docker-version' that has become dead. The marker suppresses two checks at once, so while it stood buildkit was held out of the daily freshness warning as well; removing it puts the role back in. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-db-memcached/meta/services.yml | 2 +- roles/svc-db-openldap/meta/services.yml | 2 +- roles/svc-db-redis/meta/services.yml | 2 +- roles/svc-net-tor/meta/services.yml | 2 +- roles/svc-prx-openresty/meta/services.yml | 2 +- roles/svc-runner/meta/services.yml | 3 +-- roles/web-app-bigbluebutton/meta/services.yml | 2 +- roles/web-app-fediwall/meta/services.yml | 2 +- roles/web-app-fider/meta/services.yml | 2 +- roles/web-app-jenkins/meta/services.yml | 2 +- roles/web-app-littlejs/meta/services.yml | 2 +- roles/web-app-pretix/meta/services.yml | 2 +- roles/web-app-taiga/meta/services.yml | 2 +- roles/web-app-xwiki/meta/services.yml | 2 +- roles/web-svc-coturn/meta/services.yml | 2 +- 15 files changed, 15 insertions(+), 16 deletions(-) diff --git a/roles/svc-db-memcached/meta/services.yml b/roles/svc-db-memcached/meta/services.yml index 1cd0395f56..0af5df1e7b 100644 --- a/roles/svc-db-memcached/meta/services.yml +++ b/roles/svc-db-memcached/meta/services.yml @@ -11,7 +11,7 @@ memcached: enabled: true shared: true image: memcached - version: alpine + version: 1.6.45-alpine name: memcached min_storage: 206MB backup: diff --git a/roles/svc-db-openldap/meta/services.yml b/roles/svc-db-openldap/meta/services.yml index 08111859de..0372a514db 100644 --- a/roles/svc-db-openldap/meta/services.yml +++ b/roles/svc-db-openldap/meta/services.yml @@ -12,7 +12,7 @@ openldap: shared: true provides: ldap image: debian - version: bookworm-slim + version: 12.15-slim custom: true name: openldap min_storage: 229MB diff --git a/roles/svc-db-redis/meta/services.yml b/roles/svc-db-redis/meta/services.yml index 6abd7afc9a..d8d43fba90 100644 --- a/roles/svc-db-redis/meta/services.yml +++ b/roles/svc-db-redis/meta/services.yml @@ -12,7 +12,7 @@ redis: shared: true image: redis name: redis-central - version: alpine + version: 8.10.1-alpine min_storage: 238MB backup: disabled: true diff --git a/roles/svc-net-tor/meta/services.yml b/roles/svc-net-tor/meta/services.yml index bf18fcf11d..0f872d03b9 100644 --- a/roles/svc-net-tor/meta/services.yml +++ b/roles/svc-net-tor/meta/services.yml @@ -20,7 +20,7 @@ tor: # nocheck: category-modes exclusive: true primary: true image: debian - version: trixie-slim + version: 13.6-slim custom: true name: tor node: "" diff --git a/roles/svc-prx-openresty/meta/services.yml b/roles/svc-prx-openresty/meta/services.yml index 749d07cf9e..70f57222ff 100644 --- a/roles/svc-prx-openresty/meta/services.yml +++ b/roles/svc-prx-openresty/meta/services.yml @@ -31,7 +31,7 @@ openresty: mem_reservation: 1g mem_limit: 2g image: openresty/openresty - version: alpine + version: 1.31.1.1-alpine custom: true lifecycle: beta pids_limit: 256 diff --git a/roles/svc-runner/meta/services.yml b/roles/svc-runner/meta/services.yml index c02f122847..025ae72661 100644 --- a/roles/svc-runner/meta/services.yml +++ b/roles/svc-runner/meta/services.yml @@ -30,8 +30,7 @@ buildkit: enabled: false shared: false image: moby/buildkit - # nocheck: docker-version - version: "buildx-stable-1" + version: "v0.33.0" # nocheck: dynamic-flag act-runner: enabled: false diff --git a/roles/web-app-bigbluebutton/meta/services.yml b/roles/web-app-bigbluebutton/meta/services.yml index b2985248f6..601e464673 100644 --- a/roles/web-app-bigbluebutton/meta/services.yml +++ b/roles/web-app-bigbluebutton/meta/services.yml @@ -195,7 +195,7 @@ bbb-webrtc-recorder: # nocheck: dynamic-flag: fixed upstream-compose helper, not group-gated dockerize: image: jwilder/dockerize - version: "latest" + version: "v0.11.0" enabled: false # nocheck: playwright-service-flag, dynamic-flag infrastructure backup consumer diff --git a/roles/web-app-fediwall/meta/services.yml b/roles/web-app-fediwall/meta/services.yml index 58ebaf38ed..91af39c490 100644 --- a/roles/web-app-fediwall/meta/services.yml +++ b/roles/web-app-fediwall/meta/services.yml @@ -52,7 +52,7 @@ fediwall: swarm: enabled: true image: nginx - version: alpine + version: 1.31.6-alpine custom: true upstream: version: "1.4.0" diff --git a/roles/web-app-fider/meta/services.yml b/roles/web-app-fider/meta/services.yml index 08814c2566..eab202dadd 100644 --- a/roles/web-app-fider/meta/services.yml +++ b/roles/web-app-fider/meta/services.yml @@ -36,7 +36,7 @@ fider: swarm: enabled: true image: getfider/fider - version: stable + version: v0.36.0 name: fider min_storage: 400MB backup: diff --git a/roles/web-app-jenkins/meta/services.yml b/roles/web-app-jenkins/meta/services.yml index b87b3cee70..5de40828aa 100644 --- a/roles/web-app-jenkins/meta/services.yml +++ b/roles/web-app-jenkins/meta/services.yml @@ -46,7 +46,7 @@ jenkins: # nocheck: single-replica replicas: 1 image: jenkins/jenkins - version: lts + version: 2.568.3-lts custom: true name: jenkins ports: diff --git a/roles/web-app-littlejs/meta/services.yml b/roles/web-app-littlejs/meta/services.yml index a4ad9637e4..b1b6376874 100644 --- a/roles/web-app-littlejs/meta/services.yml +++ b/roles/web-app-littlejs/meta/services.yml @@ -38,7 +38,7 @@ littlejs: repository: https://github.com/KilledByAPixel/LittleJS.git ref: main image: nginx - version: alpine + version: 1.31.6-alpine custom: true ports: local: diff --git a/roles/web-app-pretix/meta/services.yml b/roles/web-app-pretix/meta/services.yml index 9058260c92..bcccb16e2c 100644 --- a/roles/web-app-pretix/meta/services.yml +++ b/roles/web-app-pretix/meta/services.yml @@ -53,7 +53,7 @@ pretix: swarm: enabled: true image: pretix/standalone - version: stable + version: 2026.7.0 custom: true name: pretix min_storage: 1003MB diff --git a/roles/web-app-taiga/meta/services.yml b/roles/web-app-taiga/meta/services.yml index 400a570ce3..6b960b487e 100644 --- a/roles/web-app-taiga/meta/services.yml +++ b/roles/web-app-taiga/meta/services.yml @@ -92,7 +92,7 @@ gateway: internal: http: 80 image: nginx - version: alpine + version: 1.31.6-alpine cpus: "0.3" mem_reservation: 256m mem_limit: 512m diff --git a/roles/web-app-xwiki/meta/services.yml b/roles/web-app-xwiki/meta/services.yml index 0ef4b47b3a..f4cd9d48c0 100644 --- a/roles/web-app-xwiki/meta/services.yml +++ b/roles/web-app-xwiki/meta/services.yml @@ -56,7 +56,7 @@ xwiki: enabled: true # nocheck: dynamic-flag shared: true image: xwiki - version: lts-postgres-tomcat + version: 17.10.13-postgres-tomcat custom: true name: xwiki min_storage: 965MB diff --git a/roles/web-svc-coturn/meta/services.yml b/roles/web-svc-coturn/meta/services.yml index de4c0474e5..46fd83e125 100644 --- a/roles/web-svc-coturn/meta/services.yml +++ b/roles/web-svc-coturn/meta/services.yml @@ -31,7 +31,7 @@ coturn: lifecycle: beta gateway: image: nginx - version: alpine + version: 1.31.6-alpine cpus: "0.3" mem_reservation: 256m mem_limit: 512m From 10c5b35e88db5a4ed8eb6013c2d5d93af4b64cbb Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 17:00:51 +0200 Subject: [PATCH 104/197] feat(images): move seven roles off the Debian codename base onto slim A '-bookworm' tag names its Debian release by codename, which carries no ordering, so nothing can raise it across a Debian major and the base stays frozen on that release for as long as the pin stands. Moving to '-slim' drops the codename and takes the smaller image at the same time. The tag hides that this is two changes at once: node:26-slim and python:3.x-slim are trixie today, so those six roles cross Debian 12 to 13 and lose the toolchain the full buildpack-deps image carried. node:24-slim is still bookworm and its manifest is byte-identical to node:24-bookworm-slim, so flowise changes nothing today; what it gains is a tag that will follow the rebase instead of pinning Debian 12 forever. Each image was built with --no-cache and the steps read back rather than the exit code: postmarks installs git 2.47.3-0+deb13u1 and 304 npm packages, bookwyrm builds through its runtime stage so the libjpeg62-turbo, libsass1, libffi8, libxslt1.1 and libmagic1 it hardcodes do exist on trixie, roulette-wheel compiles its webpack bundle. Two were additionally exercised at runtime: sharp loads its prebuilt binding in the simpleicons image, and bridgy-fed imports secp256k1 and constructs flask_app.app. web-svc-cdn has no Dockerfile, so its build script's seven tools were checked for presence instead. bridgy-fed needs pkg-config, autoconf, automake and libtool to survive the move. secp256k1 vendors libsecp256k1 and builds it from source: without pkg-config the metadata step refuses outright, and with it autogen.sh exits 127. python:3.14-bookworm is the full buildpack-deps image and ships both; slim does not. They land in the build stage only, so the runtime image stays slim. The flowise min-storage marker still claimed a bookworm-slim base and now names what is actually built. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-bookwyrm/meta/services.yml | 2 +- roles/web-app-bridgy-fed/files/Dockerfile | 2 +- roles/web-app-bridgy-fed/meta/services.yml | 2 +- roles/web-app-flowise/meta/services.yml | 4 ++-- roles/web-app-postmarks/meta/services.yml | 2 +- roles/web-app-roulette-wheel/meta/services.yml | 2 +- roles/web-svc-cdn/meta/services.yml | 2 +- roles/web-svc-simpleicons/meta/services.yml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/web-app-bookwyrm/meta/services.yml b/roles/web-app-bookwyrm/meta/services.yml index 3bd70141b8..6bd6c005ea 100644 --- a/roles/web-app-bookwyrm/meta/services.yml +++ b/roles/web-app-bookwyrm/meta/services.yml @@ -59,7 +59,7 @@ bookwyrm: image: python # TODO: bump when upstream Bookwyrm declares support for Python >= 3.12 in requirements.txt # nocheck: docker-version - version: 3.11-bookworm + version: 3.11-slim custom: true repository: https://github.com/bookwyrm-social/bookwyrm.git ref: v0.9.3 diff --git a/roles/web-app-bridgy-fed/files/Dockerfile b/roles/web-app-bridgy-fed/files/Dockerfile index 91f07076a6..2446818fdc 100644 --- a/roles/web-app-bridgy-fed/files/Dockerfile +++ b/roles/web-app-bridgy-fed/files/Dockerfile @@ -18,7 +18,7 @@ ARG BRIDGY_REF RUN rm -rf /var/lib/apt/lists/* && \ apt-get -o Acquire::Retries=3 update && apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ - git build-essential curl ca-certificates && \ + git build-essential pkg-config autoconf automake libtool curl ca-certificates && \ rm -rf /var/lib/apt/lists/* WORKDIR /app diff --git a/roles/web-app-bridgy-fed/meta/services.yml b/roles/web-app-bridgy-fed/meta/services.yml index 41b3d862d3..1678d4b203 100644 --- a/roles/web-app-bridgy-fed/meta/services.yml +++ b/roles/web-app-bridgy-fed/meta/services.yml @@ -47,7 +47,7 @@ bridgy-fed: swarm: enabled: true image: python - version: 3.14-bookworm + version: 3.14-slim name: web-app-bridgy-fed repository: https://github.com/snarfed/bridgy-fed.git # nocheck: repository-version -- upstream publishes no tags, so the pin is a commit diff --git a/roles/web-app-flowise/meta/services.yml b/roles/web-app-flowise/meta/services.yml index a8d7cb5f01..fd68bc35fe 100644 --- a/roles/web-app-flowise/meta/services.yml +++ b/roles/web-app-flowise/meta/services.yml @@ -60,12 +60,12 @@ flowise: no_stop_required: false image: node # nocheck: docker-version - version: "24-bookworm-slim" + version: "24-slim" npm_version: 3.1.4 sqlite_version: 12.11.1 custom: true name: flowise - # nocheck: min-storage-headroom Reason: the deployed image is built here from node:24-bookworm-slim and measures 4.8 GB, so the base image transfer is not its footprint + # nocheck: min-storage-headroom Reason: the deployed image is built here from node:24-slim and measures 4.8 GB, so the base image transfer is not its footprint min_storage: 6GB cpus: "1.0" mem_reservation: 1g diff --git a/roles/web-app-postmarks/meta/services.yml b/roles/web-app-postmarks/meta/services.yml index 6dd146a1f1..fa6100d4ee 100644 --- a/roles/web-app-postmarks/meta/services.yml +++ b/roles/web-app-postmarks/meta/services.yml @@ -46,7 +46,7 @@ postmarks: swarm: enabled: true image: node - version: 26-bookworm + version: 26-slim custom: true name: web-app-postmarks repository: https://github.com/ckolderup/postmarks.git diff --git a/roles/web-app-roulette-wheel/meta/services.yml b/roles/web-app-roulette-wheel/meta/services.yml index 9dfbe51a38..f7ef64e6d9 100644 --- a/roles/web-app-roulette-wheel/meta/services.yml +++ b/roles/web-app-roulette-wheel/meta/services.yml @@ -31,7 +31,7 @@ roulette-wheel: swarm: enabled: true image: node - version: 26-bookworm + version: 26-slim custom: true repository: https://github.com/kevinveenbirkenbach/roulette-wheel.git ref: master diff --git a/roles/web-svc-cdn/meta/services.yml b/roles/web-svc-cdn/meta/services.yml index 7ec49c1f4b..d23364961d 100644 --- a/roles/web-svc-cdn/meta/services.yml +++ b/roles/web-svc-cdn/meta/services.yml @@ -30,7 +30,7 @@ build: # nocheck: single-replica one-shot npm-mirror build writing the shared served root; parallel replicas would race and duplicate work replicas: 1 image: node - version: "26-bookworm" + version: "26-slim" cpus: "1.0" mem_reservation: 128m mem_limit: 256m diff --git a/roles/web-svc-simpleicons/meta/services.yml b/roles/web-svc-simpleicons/meta/services.yml index 316dbfa8e7..4a12b9ad13 100644 --- a/roles/web-svc-simpleicons/meta/services.yml +++ b/roles/web-svc-simpleicons/meta/services.yml @@ -40,7 +40,7 @@ simpleicons: enabled: true shared: true image: node - version: 26-bookworm + version: 26-slim local_image: simpleicons-server local_version: latest min_storage: 612MB From 66be8f6a0a66a5f9920a7f4dda2042a0d3c28301 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 17:03:53 +0200 Subject: [PATCH 105/197] test(lint): refuse a Debian codename in an image tag A codename carries no ordering, so 'bookworm' and 'trixie' are two opaque strings and no updater can tell that one supersedes the other. A pin that names its Debian release that way is frozen on that release for as long as it stands, and the freeze is invisible because the tag keeps resolving. Numeric tags are comparable and stay inside the freshness pipeline, so the rule is worth holding for the whole repository rather than fixing case by case. The check reads two shapes, because a codename reaches the tree in two spellings. A 'name:tag' reference is recognised by the absence of a space after the colon, which is what separates 'debian:bookworm' from the YAML 'image: python' and covers Dockerfile FROM lines and heredocs without a second parser. A 'version:' value is read as a key, because '3.11-bookworm' carries no colon at all. False positives are kept out by two independent gates: a word-boundary prefilter, so 'residual' and 'inside' are not read as 'sid', and a split of the tag on '-', '.' and '_' that requires a whole component to be a codename. That is what lets 'cache buster', '.stretched-link', 'fa-bullseye' and a tags list containing 'stretch' through. Every codename occurrence in the repository was checked against the matcher by hand, and the check was negative-controlled in both shapes, including an untracked file. Four references had to move for the rule to hold. meta/distros.yml takes 'debian:13' rather than '13.6': at depth 1 the updater can raise it across a major, while depth 2 pins it inside the 13.x line forever. The whole 'make install' path was run in a debian:13 container to back that: all twenty-four Depends and Recommends resolve on trixie, the docker repository bootstrap derives the right suite and installs docker-ce-cli, and the package builds and installs. The workspace DiD probe moves to 'debian:13-slim'. An earlier version of this change carried a suppression claiming the probe had to follow the suite the apt proxy mirrors. That was wrong twice over: it cited a function that has nothing to do with the probe, and the coupling does not exist, because the cache frontend rewrites by path prefix and the suite never enters a routing decision. Replaying the probe verbatim against debian:13-slim fetches all three trixie suites through the bookworm-configured proxies, so the suppression is gone rather than reworded. The rule ships with an escape hatch and no uses of it. Co-Authored-By: Claude Opus 5 (1M context) --- cli/meta/ci/dev_runtime.py | 2 +- .../actions/testing/suppression.md | 1 + meta/distros.yml | 2 +- scripts/tests/workspace/utils/cache.sh | 2 +- .../test_no_debian_codename_tags.py | 222 ++++++++++++++++++ .../cli/meta/ci/test_dev_runtime_matrix.py | 4 +- 6 files changed, 228 insertions(+), 5 deletions(-) create mode 100644 tests/lint/repository/test_no_debian_codename_tags.py diff --git a/cli/meta/ci/dev_runtime.py b/cli/meta/ci/dev_runtime.py index ff4f248163..ebb196d178 100644 --- a/cli/meta/ci/dev_runtime.py +++ b/cli/meta/ci/dev_runtime.py @@ -28,7 +28,7 @@ def image_label(image: str) -> str: """Shorten a dev-runtime image to its last two registry segments. ``quay.io/centos/centos:latest`` becomes ``centos:latest``; a two-segment - image such as ``debian:bookworm`` is returned unchanged. + image such as ``debian:13`` is returned unchanged. Args: image: full image reference. diff --git a/docs/contributing/actions/testing/suppression.md b/docs/contributing/actions/testing/suppression.md index 48661f5814..5e34073123 100644 --- a/docs/contributing/actions/testing/suppression.md +++ b/docs/contributing/actions/testing/suppression.md @@ -48,6 +48,7 @@ The placement rule is per check. The catalog "Position" column uses these labels | `docker-version` | same or above | [test_image_versions.py](../../../../tests/external/update/docker/test_image_versions.py), [test_semver_pinning.py](../../../../tests/lint/ansible/services/test_semver_pinning.py) | Skips both the live version-update warning and the non-semver-pin lint for that image's `version:` key. | | `repository-version` | same or above | [test_repository_versions.py](../../../../tests/external/update/repository/test_repository_versions.py), [test_semver_pinning.py](../../../../tests/lint/ansible/services/test_semver_pinning.py) | Skips both the live ref-update warning and the non-semver-pin lint for that entity's `ref:` key. | | `dockerfile-arg-default` | same or above | [test_arg_without_default.py](../../../../tests/lint/docker/dockerfile/test_arg_without_default.py) | Allows the marked `ARG` to carry a default. The caller normally supplies every build argument so a missing one fails the build instead of selecting a base image nobody chose. | +| `debian-codename` | same or above | [test_no_debian_codename_tags.py](../../../../tests/lint/repository/test_no_debian_codename_tags.py) | Allows an image reference whose tag names a Debian release by codename. Reserved for an upstream that publishes no numeric tag; the marker MUST name which upstream, because a codename pin can never be raised by the updater. | | `direct-yaml` | same line / span | [test_no_direct_calls.py](../../../../tests/lint/repository/yaml/test_no_direct_calls.py) | Allows the marked `yaml.safe_load` / `safe_dump` call to bypass `utils.cache.yaml`. MUST use `nocheck:`, not `noqa:`. | | `curl-timeout` | same or above | [test_curl_via_filter.py](../../../../tests/lint/repository/test_curl_via_filter.py) | Allows a bare curl invocation without the `curl` filter or inline `--max-time`. | | `redundant-bool-ternary` | same line | [test_no_redundant_bool_patterns.py](../../../../tests/lint/filesystem/python/test_no_redundant_bool_patterns.py) | Allows the redundant `\| bool \| ternary('true','false')` pattern when a consumer requires the lowercase tokens. | diff --git a/meta/distros.yml b/meta/distros.yml index 0d6d719b38..977c0d8c72 100644 --- a/meta/distros.yml +++ b/meta/distros.yml @@ -46,7 +46,7 @@ distros: debian: slug: debian family: Debian - dev_runtime_image: debian:bookworm + dev_runtime_image: debian:13 galaxy_platform: Debian ubuntu: slug: ubuntu diff --git a/scripts/tests/workspace/utils/cache.sh b/scripts/tests/workspace/utils/cache.sh index 80ac36e79e..f111c74d30 100755 --- a/scripts/tests/workspace/utils/cache.sh +++ b/scripts/tests/workspace/utils/cache.sh @@ -219,7 +219,7 @@ probe_did_inner_build() { tmpdir="$(mktemp -d)" cat >"${tmpdir}/Dockerfile" <<-'EOF' - FROM debian:bookworm-slim + FROM debian:13-slim RUN apt-get update -o Acquire::http::No-Cache=true EOF tag="cache-did-probe-$$" diff --git a/tests/lint/repository/test_no_debian_codename_tags.py b/tests/lint/repository/test_no_debian_codename_tags.py new file mode 100644 index 0000000000..5445e3ab86 --- /dev/null +++ b/tests/lint/repository/test_no_debian_codename_tags.py @@ -0,0 +1,222 @@ +"""Lint: a container image tag MUST NOT name a Debian release by codename. + +Why +--- + +A codename carries no ordering. ``bookworm`` and ``trixie`` are two +opaque strings, so neither the image updater at +[test_image_versions.py](../../external/update/docker/test_image_versions.py) +nor the semver pin lint at +[test_semver_pinning.py](../../lint/ansible/services/test_semver_pinning.py) +can tell that one supersedes the other. A python image pinned to the Debian +12 codename is therefore frozen on that release for as long as it stands: no +job will ever raise it, and the freeze is invisible because the tag keeps +resolving. + +The numeric equivalent (``debian:13``, ``debian:13.6``) is comparable, so +the same pin stays inside the freshness pipeline and crosses a Debian major +the way every other version pin does. + +Scope +----- + +Every tracked file. Two shapes are recognised: + +* a ``name:tag`` image reference with no space after the colon, which + covers ``image:`` values, Dockerfile ``FROM`` lines and heredocs, +* a ``version:`` / ``*_version:`` / ``*_tag:`` YAML value, whose tag + carries no registry prefix to anchor it. + +A tag is split on ``-``, ``.`` and ``_``; a finding needs one whole +component to be a codename, so ``cache buster`` and ``.stretched-link`` +do not match. + +Out of scope: an apt suite name (``deb ... bookworm main``, a Nexus proxy +``"distribution"``). A sources.list has no numeric form, so the rule would +have no fix to offer there. + +Suppression +----------- + +``# nocheck: debian-codename`` on or above the line, for an upstream that +publishes no numeric tag. The marker names which upstream, because the pin +it buys can never be raised. +""" + +from __future__ import annotations + +import re +import subprocess +import unittest +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from utils.annotations.suppress import is_suppressed_at +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +if TYPE_CHECKING: + from pathlib import Path + +_RULE = "debian-codename" + +_CODENAMES = frozenset( + { + "squeeze", + "wheezy", + "jessie", + "stretch", + "buster", + "bullseye", + "bookworm", + "trixie", + "forky", + "duke", + "sid", + } +) + +_MENTION_RE = re.compile( + rf"(?[A-Za-z0-9_]*(?:version|tag))\s*:\s*(?P\S+)", + re.IGNORECASE, +) + +_TAG_SPLIT_RE = re.compile(r"[-._]") + + +@dataclass(frozen=True) +class Violation: + """One codename-tagged reference. + + Args: + file: repo-relative path. + line_no: 1-based line the reference sits on. + reference: the offending text as written. + codename: the release component that triggered the finding. + """ + + file: str + line_no: int + reference: str + codename: str + + +def _codename_of(tag: str) -> str | None: + """Return the codename component of *tag*, or None when it carries none. + + Args: + tag: an image tag, without the ``name:`` prefix. + """ + for component in _TAG_SPLIT_RE.split(tag.strip("'\"")): + if component.lower() in _CODENAMES: + return component.lower() + return None + + +def _references_on(line: str) -> list[tuple[str, str]]: + """Yield ``(reference, codename)`` for every codename-tagged reference. + + Args: + line: one source line already known to mention a codename. + """ + found: list[tuple[str, str]] = [] + for match in _IMAGE_REF_RE.finditer(line): + codename = _codename_of(match.group(2)) + if codename: + found.append((match.group(0), codename)) + if found: + return found + key_match = _VERSION_KEY_RE.match(line) + if key_match: + value = key_match.group("value").split("#", 1)[0] + codename = _codename_of(value) + if codename: + found.append((f"{key_match.group('key')}: {value}", codename)) + return found + + +def _git_lines(*args: str) -> list[str]: + out = subprocess.check_output( + ["git", "-c", "safe.directory=*", "-C", str(PROJECT_ROOT), *args], + text=True, + ) + return [line for line in out.splitlines() if line] + + +def _targets() -> set[str]: + """Every tracked file plus every untracked new one. + + A role added but not yet staged carries its pins like any other, so the + lint MUST see it before the commit rather than after. + """ + return set(_git_lines("ls-files")) | set( + _git_lines("ls-files", "--others", "--exclude-standard") + ) + + +def _scan_file(path: Path) -> list[Violation]: + """Return every unsuppressed codename reference in *path*. + + Args: + path: absolute path to a tracked file. + """ + try: + text = read_text(str(path)) + except (OSError, UnicodeDecodeError, ValueError): + return [] + + if not _MENTION_RE.search(text): + return [] + + rel = path.relative_to(PROJECT_ROOT).as_posix() + lines = text.splitlines() + violations: list[Violation] = [] + for line_no, raw in enumerate(lines, start=1): + if not _MENTION_RE.search(raw): + continue + for reference, codename in _references_on(raw): + if is_suppressed_at(lines, line_no, _RULE): + continue + violations.append(Violation(rel, line_no, reference, codename)) + return violations + + +class TestNoDebianCodenameTags(unittest.TestCase): + """Fails on any image tag that names a Debian release by codename.""" + + def test_no_image_tag_names_a_debian_codename(self) -> None: + violations: list[Violation] = [] + for rel in sorted(_targets()): + path = PROJECT_ROOT / rel + if path.is_file(): + violations.extend(_scan_file(path)) + + if not violations: + return + + shown = "\n".join( + f"{v.file}:{v.line_no}: {v.reference} (Debian {v.codename})" + for v in sorted(violations, key=lambda v: (v.file, v.line_no)) + ) + self.fail( + f"{len(violations)} image reference(s) pinned to a Debian codename. " + "A codename is not ordered, so no updater can raise it across a " + "Debian major and the pin stays frozen on that release forever. " + "Use the numeric tag instead (debian:13, python:3.11-slim), or " + f"declare the exception with '# nocheck: {_RULE}' naming the " + f"upstream that publishes no numeric tag:\n{shown}" + ) + + +if __name__ == "__main__": # pragma: no cover + unittest.main() diff --git a/tests/unit/python/cli/meta/ci/test_dev_runtime_matrix.py b/tests/unit/python/cli/meta/ci/test_dev_runtime_matrix.py index 4a10c8d458..bb4fd28e5e 100644 --- a/tests/unit/python/cli/meta/ci/test_dev_runtime_matrix.py +++ b/tests/unit/python/cli/meta/ci/test_dev_runtime_matrix.py @@ -7,7 +7,7 @@ import unittest from cli.meta.ci.dev_runtime import image_label, workspace_matrix from utils.symbol_glossary import to_emoji -IMAGES = ["debian:bookworm", "fedora:latest", "ubuntu:latest", "quay.io/centos:latest"] +IMAGES = ["debian:13", "fedora:latest", "ubuntu:latest", "quay.io/centos:latest"] class TestWorkspaceMatrix(unittest.TestCase): @@ -55,7 +55,7 @@ class TestWorkspaceMatrix(unittest.TestCase): def test_a_label_keeps_only_the_last_two_segments(self) -> None: self.assertEqual(image_label("quay.io/centos/centos:latest"), "centos:latest") self.assertEqual(image_label("manjarolinux/base"), "manjarolinux/base") - self.assertEqual(image_label("debian:bookworm"), "debian:bookworm") + self.assertEqual(image_label("debian:13"), "debian:13") self.assertEqual(image_label("ghcr.io/foo/bar/baz:1.2"), "baz:1.2") self.assertEqual(image_label("plainname"), "plainname") From 23fcad5ddfd456b4a98ec29e8c770b7b7f47ac3e Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 17:43:13 +0200 Subject: [PATCH 106/197] feat(update): read a release line written before the number as a family Upstreams that ship several channels in parallel put the channel first: LiteLLM publishes 'main-v1.77.7-stable' next to 'main-v1.77.7-nightly', Jitsi ships 'stable-9646'. The grammar only allowed a leading 'v', so both shapes parsed as nothing at all, sat at depth 0 next to 'latest', and the bump job never looked at them. The numeric core now takes an optional leading channel, and the channel folds into the flavor the same way the patch letter already does. Everything that is not the number is family; only the number is order. That is what keeps '-stable' away from '-nightly' and 'stable-9646' away from a bare '9646', which would leave the channel and change what the role deploys. The change is additive. Only a tag starting with letters followed by a dash can match the new group, and every such tag failed the grammar outright before, so no tag that parses today parses differently. The regression case pins the four shapes that already worked, and the new cases pin the two directions that must not happen along with the tags that must stay refused: 'act-latest', 'bookworm-slim', 'main-latest' and 'buildx-stable-1' carry a channel but no version. Two pins in this repository become orderable as a result: the four Jitsi components and the LiteLLM gateway. Co-Authored-By: Claude Opus 5 (1M context) --- .../utils/update/test_base_pin_families.py | 42 +++++++++++++++++++ utils/update/base.py | 31 +++++++++----- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/tests/unit/python/utils/update/test_base_pin_families.py b/tests/unit/python/utils/update/test_base_pin_families.py index fe53566fd6..564fcab71a 100644 --- a/tests/unit/python/utils/update/test_base_pin_families.py +++ b/tests/unit/python/utils/update/test_base_pin_families.py @@ -97,5 +97,47 @@ class TestNothingElseChanged(unittest.TestCase): ) +class TestChannelPrefixedTags(unittest.TestCase): + """A release line written before the number is a family, not a version.""" + + def test_a_channel_prefixed_tag_is_orderable(self): + self.assertTrue(is_semver("main-v1.77.7-stable")) + self.assertEqual(version_depth("main-v1.77.7-stable"), 3) + self.assertEqual(version_key("main-v1.77.7-stable"), (1, 77, 7, 0)) + + def test_it_bumps_inside_its_own_channel(self): + tags = ["main-v1.77.7-stable", "main-v1.83.14-stable"] + self.assertEqual(_latest("main-v1.77.7-stable", tags), "main-v1.83.14-stable") + + def test_it_never_crosses_into_a_sibling_channel(self): + tags = ["main-v1.83.14-nightly", "main-v1.83.14-stable"] + self.assertEqual(_latest("main-v1.77.7-stable", tags), "main-v1.83.14-stable") + self.assertEqual(_latest("main-v1.77.7-nightly", tags), "main-v1.83.14-nightly") + + def test_a_bare_number_is_a_different_family_than_a_channelled_one(self): + self.assertNotEqual(version_flavor("stable-9646"), version_flavor("9646")) + self.assertEqual( + _latest("stable-9646", ["stable-9646", "9999"]), + "stable-9646", + "a bare build number leaves the stable channel and would change " + "what the role deploys", + ) + + def test_it_bumps_inside_the_stable_channel(self): + self.assertEqual( + _latest("stable-9646", ["stable-9646", "stable-9999"]), "stable-9999" + ) + + def test_a_channel_without_a_number_stays_refused(self): + for tag in ("act-latest", "bookworm-slim", "main-latest", "buildx-stable-1"): + self.assertFalse(is_semver(tag), tag) + + def test_a_date_stamped_tag_orders_as_one_number(self): + self.assertEqual(version_depth("20260917"), 1) + self.assertEqual( + _latest("20260702", ["20260702", "20260706", "20260917"]), "20260917" + ) + + if __name__ == "__main__": unittest.main() diff --git a/utils/update/base.py b/utils/update/base.py index bd10ac796f..161664e813 100644 --- a/utils/update/base.py +++ b/utils/update/base.py @@ -1,18 +1,24 @@ """Semver primitives shared by every version-bump backend. A "version" here is a tag of the shape -``v??(-)?``, where the numeric part has 1 to 5 -dot-separated components, the optional ```` is a vendor counter written -as a letter and a number (e.g. the Checkmk tag ``2.4.0p32``), and the optional -``-`` suffix is an opaque discriminator (e.g. the Docker Official Image -tag ``5.4.5-php8.3-apache`` or the npm-style ``1.2.3-alpha``). +``(-)?v??(-)?``, where the numeric part +has 1 to 5 dot-separated components, the optional leading ``-`` is a +release line written before the number (e.g. the LiteLLM tag +``main-v1.77.7-stable`` or the Jitsi tag ``stable-9646``), the optional +```` is a vendor counter written as a letter and a number (e.g. the +Checkmk tag ``2.4.0p32``), and the optional ``-`` suffix is an opaque +discriminator (e.g. the Docker Official Image tag ``5.4.5-php8.3-apache`` or +the npm-style ``1.2.3-alpha``). Upgrade candidates MUST share the same depth (1 to 5 components) AND the same flavor as the current tag, so that ``5.4.5-php8.3-apache`` never silently bumps to ``5.4.6-php8.4-apache`` (different runtime) or to ``5.4.6`` (different -depth). The patch letter joins the flavor and its number joins the ordering -key, which is what keeps ``2.4.0p32`` ordering against ``2.4.0p33`` and never -against the four-component ``2.4.0.32``. +depth). Everything that is not the number folds into the flavor: the leading +channel, the trailing suffix and the patch letter. That is what keeps +``main-v1.77.7-stable`` away from ``main-v1.77.7-nightly``, ``stable-9646`` +away from a bare ``9646``, and ``2.4.0p32`` ordering against ``2.4.0p33`` +rather than against the four-component ``2.4.0.32``. Only the patch number +joins the ordering key, so ``p9`` sorts below ``p10``. """ from __future__ import annotations @@ -22,7 +28,8 @@ import re _SEMVER_CORE = r"v?\d+(?:\.\d+){0,4}" _VERSIONED_TAG_RE = re.compile( - rf"^(?P{_SEMVER_CORE})(?P[A-Za-z]\d+)?(?P-\S+)?$" + rf"^(?P[A-Za-z][A-Za-z0-9]*-)?(?P{_SEMVER_CORE})" + rf"(?P[A-Za-z]\d+)?(?P-\S+)?$" ) @@ -31,7 +38,11 @@ def _parse_versioned_tag(tag: str) -> tuple[str, str, int | None] | None: if match is None: return None patch = match.group("patch") - flavor = (match.group("flavor") or "") + (patch[0] if patch else "") + flavor = ( + (match.group("channel") or "") + + (match.group("flavor") or "") + + (patch[0] if patch else "") + ) return match.group("semver"), flavor, int(patch[1:]) if patch else None From e66005ab073f9eafef3c0f4d2c296960f9cf6abe Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 17:45:28 +0200 Subject: [PATCH 107/197] fix(update): move a role's images as one release or not at all Seven roles pin several images at one shared version, because those images are one product: the four Jitsi components, six GitLab CNG images, eight BigBlueButton services, three Penpot, plus Mastodon, Funkwhale and Nextcloud Talk. The bump job walks entries one by one and takes each image's own newest tag, so a run landing between upstream's first and last push raises the components already there and leaves the rest behind. What deploys afterwards is a mixed set, and for Jitsi a mixed set breaks video. The window is real rather than theoretical. Measured on Docker Hub, 'stable-9457' exists for jitsi/prosody, jitsi/jicofo and jitsi/jvb but not for jitsi/web, one build out of sixty. Candidates are now grouped by role and current version, and the group is held to the newest tag the intersection of all their tag lists still offers. A group with nothing newer in common yields no update at all rather than a partial one. A lone entry passes through untouched, so nothing changes for the large majority. The key is deliberately the role and the version, not the image namespace. A namespace rule does not survive the data: prom/* carries four components at four different versions, opentalk/* three, and minio, taigaio and markoshust each two. Grouping by namespace would hold back pins that are correctly independent. Five of the seven roles have carried this exposure all along; the channel grammar in the previous commit added the sixth by making Jitsi orderable at all. This closes all of them. Co-Authored-By: Claude Opus 5 (1M context) --- .../update/test_docker_release_groups.py | 113 ++++++++++++++++++ utils/update/docker.py | 49 +++++++- 2 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 tests/unit/python/utils/update/test_docker_release_groups.py diff --git a/tests/unit/python/utils/update/test_docker_release_groups.py b/tests/unit/python/utils/update/test_docker_release_groups.py new file mode 100644 index 0000000000..11fd767a6a --- /dev/null +++ b/tests/unit/python/utils/update/test_docker_release_groups.py @@ -0,0 +1,113 @@ +"""Several images of one role at one version move together or not at all. + +Upstream does not push a release group atomically. Measured on Docker Hub: +``jitsi/prosody``, ``jitsi/jicofo`` and ``jitsi/jvb`` all publish +``stable-9457`` while ``jitsi/web`` does not, so a bump run landing in that +window would raise three components and leave the fourth behind. +""" + +from __future__ import annotations + +import unittest +from pathlib import Path + +from utils.update.docker import ( + DockerImageVersionEntry, + DockerImageVersionUpdate, + _held_to_release_groups, +) + +JITSI = ("web", "prosody", "jicofo", "jvb") + + +def _entry(role: str, service: str, image: str, version: str): + return DockerImageVersionEntry( + role=role, + service=service, + image=image, + version=version, + config_path=Path("meta/services.yml"), + ) + + +def _jitsi_updates(version: str, latest: str): + return [ + DockerImageVersionUpdate( + entry=_entry("web-app-jitsi", name, f"jitsi/{name}", version), + latest=latest, + ) + for name in JITSI + ] + + +class TestReleaseGroupsMoveTogether(unittest.TestCase): + def test_a_partial_publish_holds_the_whole_group_back(self): + tags = { + "jitsi/web": ["stable-9400", "stable-9646"], + "jitsi/prosody": ["stable-9400", "stable-9646", "stable-9700"], + "jitsi/jicofo": ["stable-9400", "stable-9646", "stable-9700"], + "jitsi/jvb": ["stable-9400", "stable-9646", "stable-9700"], + } + updates = _jitsi_updates("stable-9400", "stable-9700") + + held = _held_to_release_groups(updates, tags) + + self.assertEqual({update.latest for update in held}, {"stable-9646"}) + self.assertEqual(len(held), 4) + + def test_a_group_all_publish_moves_to_the_head(self): + tags = { + f"jitsi/{name}": ["stable-9400", "stable-9700"] for name in JITSI + } + updates = _jitsi_updates("stable-9400", "stable-9700") + + held = _held_to_release_groups(updates, tags) + + self.assertEqual({update.latest for update in held}, {"stable-9700"}) + + def test_a_group_with_nothing_in_common_is_dropped(self): + tags = { + "jitsi/web": ["stable-9400", "stable-9701"], + "jitsi/prosody": ["stable-9400", "stable-9702"], + "jitsi/jicofo": ["stable-9400", "stable-9703"], + "jitsi/jvb": ["stable-9400", "stable-9704"], + } + updates = _jitsi_updates("stable-9400", "stable-9704") + + self.assertEqual(_held_to_release_groups(updates, tags), []) + + def test_a_lone_entry_is_untouched(self): + update = DockerImageVersionUpdate( + entry=_entry("web-app-fider", "fider", "getfider/fider", "v0.35.0"), + latest="v0.36.0", + ) + + held = _held_to_release_groups([update], {"getfider/fider": ["v0.36.0"]}) + + self.assertEqual(held, [update]) + + def test_two_roles_on_one_version_are_not_one_group(self): + tags = { + "jitsi/web": ["stable-9400", "stable-9700"], + "other/thing": ["stable-9400"], + } + updates = [ + DockerImageVersionUpdate( + entry=_entry("web-app-jitsi", "web", "jitsi/web", "stable-9400"), + latest="stable-9700", + ), + DockerImageVersionUpdate( + entry=_entry("web-app-other", "thing", "other/thing", "stable-9400"), + latest="stable-9400", + ), + ] + + held = _held_to_release_groups(updates, tags) + + self.assertEqual( + [update.latest for update in held], ["stable-9700", "stable-9400"] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/update/docker.py b/utils/update/docker.py index 0c96b20cd4..53bf9561f8 100644 --- a/utils/update/docker.py +++ b/utils/update/docker.py @@ -19,6 +19,7 @@ import re import time import urllib.error import urllib.request +from collections import defaultdict from dataclasses import dataclass from typing import TYPE_CHECKING from urllib.parse import quote, urlencode @@ -342,7 +343,53 @@ def find_outdated_updates(repo_root: Path) -> list[DockerImageVersionUpdate]: if latest and version_key(entry.version) < version_key(latest): updates.append(DockerImageVersionUpdate(entry=entry, latest=latest)) - return updates + return _held_to_release_groups(updates, image_tags) + + +def _held_to_release_groups( + updates: list[DockerImageVersionUpdate], image_tags: dict[str, list[str]] +) -> list[DockerImageVersionUpdate]: + """Hold every member of a release group to the newest tag all of them + publish. + + Several images of one role that currently sit on the same version are one + product released together (the four Jitsi components, the six GitLab CNG + images). Upstream does not push them atomically, so a run that lands in the + window between the first and the last push would raise the ones already + there and leave the rest behind, deploying a mixed set. + + Args: + updates: the per-entry candidates, before grouping. + image_tags: every tag fetched per image, keyed as in the caller. + + Returns: + The updates, each group lowered to its common newest tag, and dropped + entirely where the group shares no newer tag at all. + """ + groups: dict[tuple[str, str], list[DockerImageVersionUpdate]] = defaultdict(list) + for update in updates: + groups[(update.entry.role, update.entry.version)].append(update) + + held: list[DockerImageVersionUpdate] = [] + for group in groups.values(): + if len(group) == 1: + held.extend(group) + continue + shared = set.intersection( + *(set(image_tags.get(update.entry.image, [])) for update in group) + ) + current = group[0].entry.version + common = latest_semver( + sorted(shared), version_depth(current), version_flavor(current) + ) + if not common or version_key(current) >= version_key(common): + continue + held.extend( + DockerImageVersionUpdate(entry=update.entry, latest=common) + for update in group + ) + + return held def update_config_versions(config_path: Path, service_versions: dict[str, str]) -> bool: From e14e73fba0faefed34cb87196cbcd3b23bf319da Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 17:47:02 +0200 Subject: [PATCH 108/197] refactor(docker): let a moving tag declare itself with the marker it already has A pin outside semver was given its own 'docker-ref' marker on the theory that every class needs one. That holds for a digest and not for a tag. A digest pin declares something about the registry, that bytes move under a standing tag, which no other choice says. A tag outside semver declares that upstream publishes nothing orderable, which is exactly what 'docker-version' already means and what minio and the act runner already carry on their version lines. Keeping both would have put '# nocheck: docker-version, docker-ref' on every such line: two keys, two checks, one sentence said twice. 'docker-ref' is dropped instead. It lived in this one mapping, was never entered in the suppression catalog, and no check ever read it. The semver entry goes with it, because a semver pin never needs a declaration. The test that asserted three distinct rules is replaced by two that pin the separation where it is real: a digest marker differs from a tag marker, and a semver pin has no marker at all. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/python/utils/docker/test_image_pin.py | 15 +++++++++++---- utils/docker/image/pin.py | 3 +-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/unit/python/utils/docker/test_image_pin.py b/tests/unit/python/utils/docker/test_image_pin.py index 1b7b7e949f..a8737c8157 100644 --- a/tests/unit/python/utils/docker/test_image_pin.py +++ b/tests/unit/python/utils/docker/test_image_pin.py @@ -33,11 +33,18 @@ class TestPinClass(unittest.TestCase): self.assertEqual(pin_class(f" {DIGEST_VALUE} "), DIGEST) self.assertEqual(pin_class(" 3.0.1 "), SEMVER) - def test_every_class_has_its_own_rule(self): - rules = [RULE_BY_CLASS[SEMVER], RULE_BY_CLASS[DIGEST], RULE_BY_CLASS[REF]] + def test_a_digest_declares_itself_apart_from_a_tag(self): + self.assertNotEqual( + RULE_BY_CLASS[DIGEST], + RULE_BY_CLASS[REF], + "a digest is chosen for a reason a non-semver tag never carries, so " + "the two cannot share one marker", + ) + self.assertEqual(RULE_BY_CLASS[DIGEST], "docker-digest") + self.assertEqual(RULE_BY_CLASS[REF], "docker-version") - self.assertEqual(len(set(rules)), 3, "a shared rule cannot express intent") - self.assertEqual(rules, ["docker-version", "docker-digest", "docker-ref"]) + def test_a_semver_pin_needs_no_declaration(self): + self.assertNotIn(SEMVER, RULE_BY_CLASS) class TestPullReference(unittest.TestCase): diff --git a/utils/docker/image/pin.py b/utils/docker/image/pin.py index 4ddfce88e5..11b4ec7e41 100644 --- a/utils/docker/image/pin.py +++ b/utils/docker/image/pin.py @@ -7,9 +7,8 @@ DIGEST = "digest" REF = "ref" RULE_BY_CLASS = { - SEMVER: "docker-version", DIGEST: "docker-digest", - REF: "docker-ref", + REF: "docker-version", } From 0c018fa31850739d21626925c894ce8081f2f77b Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 17:50:27 +0200 Subject: [PATCH 109/197] test(update): require every image pin to be orderable or declared The ranking the last commits built toward is now enforced: a semver tag, else a digest, and nothing else without a written reason. Two states fail. A pin that is neither names something the registry can move underneath us, and nothing here can tell whether it still points where it did when it was written. A digest on an upstream that does publish semver takes the fallback for no gain and hides the entry from the update job. Thirteen entries stood in the way when the check was first run. Four were resolved by the channel grammar, which made the Jitsi components orderable without touching their pins. Three needed nothing once a moving tag could declare itself with 'docker-version'. Two are fixed here and four are declared. LiteLLM leaves 'main-v1.77.3.dynamic_rates'. That tag exists exactly once among 2609, so it is a dead end no updater can follow, and it misreports itself: the image inside answers 1.77.2. The pin moves to 'main-v1.77.7-stable', five patches along the same minor and onto a channel with 134 builds. Started against this repository's config shape, 1.77.7 registers both model entry forms and accepts drop_params and the os.environ master key. Pixelfed leaves 'latest' for '20260917'. The upstream stamps its dates as one eight-digit number, which is depth 1 and orderable with no tooling change at all, and the two tags resolve to the same digest, so nothing about the deployed image moves. The four declarations carry the count that justifies them rather than a claim: LM Studio publishes two tags and neither is a version, the MIG registry carries one, and the two Nextcloud Talk images publish 174 and 104 AIO build stamps between them. Both branches of the gate were negative-controlled. The digest branch skips in this repository because no digest pin exists at all, so it was exercised by introducing one and watching it fail. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-litellm/meta/services.yml | 2 +- roles/svc-ai-lmstudio/meta/services.yml | 1 + roles/web-app-mig/meta/services.yml | 1 + roles/web-app-nextcloud/meta/services.yml | 2 + roles/web-app-pixelfed/meta/services.yml | 2 +- .../update/docker/test_image_pin_policy.py | 123 ++++++++++++++++++ 6 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 tests/external/update/docker/test_image_pin_policy.py diff --git a/roles/svc-ai-litellm/meta/services.yml b/roles/svc-ai-litellm/meta/services.yml index 5d7177968c..663f5e67c0 100644 --- a/roles/svc-ai-litellm/meta/services.yml +++ b/roles/svc-ai-litellm/meta/services.yml @@ -11,7 +11,7 @@ litellm: backup: no_stop_required: true image: ghcr.io/berriai/litellm - version: main-v1.77.3.dynamic_rates + version: main-v1.77.7-stable name: litellm max_output_tokens: 512 upstream_timeout: 240 diff --git a/roles/svc-ai-lmstudio/meta/services.yml b/roles/svc-ai-lmstudio/meta/services.yml index b9a3f271ed..d27bb1b516 100644 --- a/roles/svc-ai-lmstudio/meta/services.yml +++ b/roles/svc-ai-lmstudio/meta/services.yml @@ -11,6 +11,7 @@ lmstudio: backup: no_stop_required: true image: lmstudio/llmster-preview + # nocheck: docker-version -- the repository publishes 2 tags and neither is a version; 'cpu' selects the backend, not a release version: cpu name: lmstudio platform_arch: x86_64 diff --git a/roles/web-app-mig/meta/services.yml b/roles/web-app-mig/meta/services.yml index a21a0d5290..feb3a06616 100644 --- a/roles/web-app-mig/meta/services.yml +++ b/roles/web-app-mig/meta/services.yml @@ -34,6 +34,7 @@ mig: swarm: enabled: true image: ghcr.io/kevinveenbirkenbach/infinito-mig + # nocheck: docker-version -- this registry carries exactly 1 tag; the image is built from this project and has no release line yet version: latest name: mig min_storage: 1224MB diff --git a/roles/web-app-nextcloud/meta/services.yml b/roles/web-app-nextcloud/meta/services.yml index 91bdeb0457..228a5b4300 100644 --- a/roles/web-app-nextcloud/meta/services.yml +++ b/roles/web-app-nextcloud/meta/services.yml @@ -197,6 +197,7 @@ talk: shared: "{{ 'web-svc-coturn' in group_names }}" name: nextcloud-talk image: nextcloud/aio-talk + # nocheck: docker-version -- all 174 tags upstream publishes are AIO build stamps, none is a version version: latest backup: no_stop_required: false @@ -271,6 +272,7 @@ talk_recording: shared: false name: nextcloud-talk-recording image: nextcloud/aio-talk-recording + # nocheck: docker-version -- all 104 tags upstream publishes are AIO build stamps, none is a version version: latest backup: no_stop_required: true diff --git a/roles/web-app-pixelfed/meta/services.yml b/roles/web-app-pixelfed/meta/services.yml index fe9b2eb8d0..222a5ee5b1 100644 --- a/roles/web-app-pixelfed/meta/services.yml +++ b/roles/web-app-pixelfed/meta/services.yml @@ -49,7 +49,7 @@ pixelfed: shared: true enabled: true # nocheck: dynamic-flag image: zknt/pixelfed - version: latest + version: "20260917" name: pixelfed min_storage: 588MB backup: diff --git a/tests/external/update/docker/test_image_pin_policy.py b/tests/external/update/docker/test_image_pin_policy.py new file mode 100644 index 0000000000..e9c572aa60 --- /dev/null +++ b/tests/external/update/docker/test_image_pin_policy.py @@ -0,0 +1,123 @@ +"""Enforce the pin ranking: a semver tag, else a digest, and nothing else. + +Two states fail here, and neither is a matter of taste: + +* A pin that is neither a semver tag nor a digest names something the registry + can move under us. Nothing in this repository can tell whether it still points + where it did when it was written. +* A digest where the registry does publish semver tags. The digest is the + fallback for upstreams that offer no orderable tag; taking it anyway opts the + entry out of the freshness pipeline for no gain. + +Both are declarable when the choice is deliberate, with ``# nocheck: +docker-version`` respectively ``# nocheck: docker-digest`` on the ``version:`` +line or the one above it. The mirror case is the reason the second one exists: +a tag whose bytes moved is worth pinning past, even though the tag exists. + +Opt-in external test: it reads live third-party registries and runs only under +the external suite. The offline sibling at +[test_semver_pinning.py](../../../lint/ansible/services/test_semver_pinning.py) +warns about the same pins without failing. +""" + +from __future__ import annotations + +import concurrent.futures +import unittest + +from utils.annotations.suppress import is_suppressed_at +from utils.cache.files import read_text +from utils.docker.image.discovery import iter_role_images +from utils.docker.image.pin import DIGEST, REF, RULE_BY_CLASS, pin_class +from utils.docker.registry import fetch_registry_tags +from utils.roles.mapping import ROLE_FILE_META_SERVICES +from utils.update.base import is_semver, resolve_max_fetch_workers +from utils.update.docker import is_supported_registry + +from . import PROJECT_ROOT + + +def _pull_image(ref) -> str: + return ref.name if ref.registry == "docker.io" else f"{ref.registry}/{ref.name}" + + +def _declared(ref, rule: str) -> bool: + """Whether the ``version:`` line carrying this pin opts out of *rule*.""" + path = PROJECT_ROOT / "roles" / ref.role / ROLE_FILE_META_SERVICES + lines = read_text(path).splitlines() + needle = ref.version.strip() + numbers = [ + number + for number, line in enumerate(lines, start=1) + if line.split("#", 1)[0].strip().startswith("version:") + and needle in line.split("#", 1)[0] + ] + return bool(numbers) and all( + is_suppressed_at(lines, number, rule) for number in numbers + ) + + +def _offers_semver(image: str) -> bool: + return any(is_semver(tag) for tag in fetch_registry_tags(image)) + + +class TestImagePinPolicy(unittest.TestCase): + """A pin is a semver tag, a justified digest, or a declared exception.""" + + def test_unpinned_and_unversioned_entries_are_rejected(self) -> None: + refs = list(iter_role_images(PROJECT_ROOT)) + self.assertTrue(refs, "No role images discovered") + + offenders = [ + f" roles/{ref.role}/{ROLE_FILE_META_SERVICES}: {ref.service} pins " + f"{ref.version!r}, which is neither a version nor a digest" + for ref in refs + if pin_class(ref.version) == REF + and not _declared(ref, RULE_BY_CLASS[REF]) + ] + + if offenders: + listing = "\n".join(sorted(offenders)) + self.fail( + "These entries pin a name the registry can move under them. " + "Pin a semver tag, or a digest when upstream publishes none, or " + f"declare the choice with `# nocheck: {RULE_BY_CLASS[REF]}`:\n" + f"{listing}" + ) + + def test_a_digest_is_rejected_where_semver_is_published(self) -> None: + refs = [ + ref + for ref in iter_role_images(PROJECT_ROOT) + if pin_class(ref.version) == DIGEST + and is_supported_registry(ref.name) + and not _declared(ref, RULE_BY_CLASS[DIGEST]) + ] + if not refs: + self.skipTest("no undeclared digest pins to check") + + images = {_pull_image(ref) for ref in refs} + with concurrent.futures.ThreadPoolExecutor( + max_workers=resolve_max_fetch_workers() + ) as pool: + offers = dict(zip(images, pool.map(_offers_semver, images), strict=True)) + + offenders = [ + f" roles/{ref.role}/{ROLE_FILE_META_SERVICES}: {ref.service} pins a " + f"digest while {_pull_image(ref)} publishes semver tags" + for ref in refs + if offers.get(_pull_image(ref)) + ] + + if offenders: + listing = "\n".join(sorted(offenders)) + self.fail( + "A digest is the fallback for an upstream that publishes no " + "orderable tag. These have one, so the digest only hides them " + f"from the update job. Use the tag, or declare why not with " + f"`# nocheck: {RULE_BY_CLASS[DIGEST]}`:\n{listing}" + ) + + +if __name__ == "__main__": + unittest.main() From badfe72a4f60d23af3b7b25d6717b2c91741ace7 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 19:26:41 +0200 Subject: [PATCH 110/197] test(update): build the release-group fixture from the role-file constant test_file_spot failed the gate on the entry fixture, which spelled meta/services.yml out by hand. The path now comes from utils.roles.mapping as ROLE_FILE_META_SERVICES, following 4a0fca2bcb, which fixed the same lint on the addon fixture next door. Co-Authored-By: Claude Opus 5 (1M context) --- .../unit/python/utils/update/test_docker_release_groups.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/unit/python/utils/update/test_docker_release_groups.py b/tests/unit/python/utils/update/test_docker_release_groups.py index 11fd767a6a..72b6c22256 100644 --- a/tests/unit/python/utils/update/test_docker_release_groups.py +++ b/tests/unit/python/utils/update/test_docker_release_groups.py @@ -11,6 +11,7 @@ from __future__ import annotations import unittest from pathlib import Path +from utils.roles.mapping import ROLE_FILE_META_SERVICES from utils.update.docker import ( DockerImageVersionEntry, DockerImageVersionUpdate, @@ -26,7 +27,7 @@ def _entry(role: str, service: str, image: str, version: str): service=service, image=image, version=version, - config_path=Path("meta/services.yml"), + config_path=Path(ROLE_FILE_META_SERVICES), ) @@ -56,9 +57,7 @@ class TestReleaseGroupsMoveTogether(unittest.TestCase): self.assertEqual(len(held), 4) def test_a_group_all_publish_moves_to_the_head(self): - tags = { - f"jitsi/{name}": ["stable-9400", "stable-9700"] for name in JITSI - } + tags = {f"jitsi/{name}": ["stable-9400", "stable-9700"] for name in JITSI} updates = _jitsi_updates("stable-9400", "stable-9700") held = _held_to_release_groups(updates, tags) From 9b5d4875f7933ac6dfd287062f30d7c6a23738c1 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 19:29:32 +0200 Subject: [PATCH 111/197] fix(hlth-csp): take the csp-checker build that stops failing on a 204 n8n ships isbot.spawn(['bot']) unconditionally and answers any user agent matching /bot/i with a bodyless 204. This checker calls itself CSP-CheckerBot, so the probe never reached the application. Chromium cancels a navigation on 204, page.goto threw net::ERR_ABORTED, and the checker reported the host unreachable. In CI job 105068914482 the unit failed 25 times over 1551 seconds and took the job with it. 3.0.2 ends the visit as reachable when the document response is exactly 204, and prints that it checked nothing, because page.url is about:blank afterwards. Exactly 204, not the 2xx range: a 200 carrying Content-Disposition aborts the same way and is a download, not an empty document. The checker's e2e suite pins both directions, with the download fixture as the control that fails if the condition is ever widened. Same shape as df4b7757bc took for 304: a status the checker's own contract calls healthy, rejected at runtime over how the browser reacts to it. The user agent keeps its name. The checker is a bot and a bot filter blocking it is correct; what changes is that the consequence is now stated rather than hidden. A vhost answering 204 reports that its CSP went unchecked instead of reporting nothing at all. Co-Authored-By: Claude Opus 5 (1M context) --- roles/sys-ctl-hlth-csp/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sys-ctl-hlth-csp/meta/services.yml b/roles/sys-ctl-hlth-csp/meta/services.yml index 90ffed8219..90cae220c6 100644 --- a/roles/sys-ctl-hlth-csp/meta/services.yml +++ b/roles/sys-ctl-hlth-csp/meta/services.yml @@ -8,4 +8,4 @@ csp: categories: [web] csp-checker: image: ghcr.io/kevinveenbirkenbach/csp-checker - version: 3.0.1 + version: 3.0.2 From c693f4f619d4ed0cfc95aa169665f5ca99f2b109 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 19:35:05 +0200 Subject: [PATCH 112/197] style(quality): apply what the gate wrote back into the tree Nothing here was typed by hand. The docs generator rebuilt the MCP audit table against pins that had already landed, so fider reads v0.36.0 instead of stable, jenkins 2.568.3-lts instead of lts, pretix 2026.7.0 instead of stable, and flowise 24-slim instead of 24-bookworm-slim. That is the table catching up with da665ab69 and 10c5b35e8, reading the same declaration the updater reads. Autoformat wrapped three test files from the same series: one condition pulled onto a single line, three assertEqual calls broken across lines. Co-Authored-By: Claude Opus 5 (1M context) --- docs/contributing/design/role/services/mcp/audit.md | 8 ++++---- tests/external/update/docker/test_image_pin_policy.py | 3 +-- tests/unit/python/plugins/filter/test_redirect_sources.py | 4 +++- tests/unit/python/utils/docker/test_image_pin.py | 8 ++++++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/contributing/design/role/services/mcp/audit.md b/docs/contributing/design/role/services/mcp/audit.md index 8fd7c84b0f..31dcaf1dbc 100644 --- a/docs/contributing/design/role/services/mcp/audit.md +++ b/docs/contributing/design/role/services/mcp/audit.md @@ -15,14 +15,14 @@ not the version the MCP surface was verified against. | `svc-db-qdrant` | v1.19.1 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-svc-db-qdrant`) | read-only | 2 | adapter | `web-app-flowise`, `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-baserow` | 2.3.3 | [link](https://baserow.io/user-docs/mcp-server) | streamable_http | bearer_token | service_account (`mcp-web-app-baserow`) | read-only | 4 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-checkmk` | 2.4.0p32 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-checkmk`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | -| `web-app-fider` | stable | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-fider`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | -| `web-app-flowise` | 24-bookworm-slim | [link](https://github.com/FlowiseAI/Flowise/tree/flowise%403.1.4/packages/server/src/routes/custom-mcp-servers) | โ€” | โ€” | โ€” | read-only | โ€” | native | โ€” | โ€” | +| `web-app-fider` | v0.36.0 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-fider`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | +| `web-app-flowise` | 24-slim | [link](https://github.com/FlowiseAI/Flowise/tree/flowise%403.1.4/packages/server/src/routes/custom-mcp-servers) | โ€” | โ€” | โ€” | read-only | โ€” | native | โ€” | โ€” | | `web-app-gitea` | 1.27.3 | [link](https://pkg.go.dev/gitea.com/gitea/gitea-mcp) | streamable_http | bearer_token | service_account (`mcp-web-app-gitea`) | read-only | 24 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-gitlab` | โ€” | [link](https://docs.gitlab.com/user/gitlab_duo/model_context_protocol/mcp_server/) | streamable_http | bearer_token | service_account (`mcp-web-app-gitlab`) | read-only | 14 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-hermes` | v2026.9.7 | [link](https://hermes-agent.nousresearch.com/) | โ€” | โ€” | โ€” | read-only | โ€” | native | โ€” | โ€” | | `web-app-homeassistant` | 2026.7 | [link](https://www.home-assistant.io/integrations/mcp_server/) | streamable_http | bearer_token | service_account (`mcp-web-app-homeassistant`) | read-only | 10 | native | `web-app-flowise`, `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-jellyfin` | 10.11.11 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-jellyfin`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | -| `web-app-jenkins` | lts | [link](https://plugins.jenkins.io/mcp-server/) | streamable_http | bearer_token | service_account (`mcp-web-app-jenkins`) | read-only | 15 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | +| `web-app-jenkins` | 2.568.3-lts | [link](https://plugins.jenkins.io/mcp-server/) | streamable_http | bearer_token | service_account (`mcp-web-app-jenkins`) | read-only | 15 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-listmonk` | v6.2.0 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-listmonk`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-mattermost` | 11.11.0 | [link](https://github.com/mattermost/mattermost-plugin-agents/blob/master/mcpserver/README.md) | streamable_http | bearer_token | service_account (`mcp-web-app-mattermost`) | read-only | 11 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-moodle` | 8.3-fpm | [link](https://moodle.org/plugins/webservice_mcp) | streamable_http | bearer_token | service_account (`mcp-web-app-moodle`) | read-only | 5 | plugin | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | @@ -30,7 +30,7 @@ not the version the MCP surface was verified against. | `web-app-nextcloud` | 34-fpm-alpine | [link](https://docs.nextcloud.com/server/latest/admin_manual/ai/app_context_agent.html) | streamable_http | bearer_token | service_account (`mcp-web-app-nextcloud`) | read-only | 9 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-openclaw` | 2026.7.1 | [link](https://openclaw.ai/) | โ€” | โ€” | โ€” | read-only | โ€” | native | โ€” | โ€” | | `web-app-openwebui` | 0.11.0 | [link](https://docs.openwebui.com/features/extensibility/mcp/) | โ€” | โ€” | โ€” | read-only | โ€” | native | โ€” | โ€” | -| `web-app-pretix` | stable | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-pretix`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | +| `web-app-pretix` | 2026.7.0 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-pretix`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-prometheus` | v3.14.0 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-prometheus`) | read-only | 3 | adapter | `web-app-flowise`, `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-snipe-it` | โ€” | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-snipe-it`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-wordpress` | 7.1.0 | [link](https://github.com/WordPress/mcp-adapter/tree/v0.5.0) | streamable_http | basic_auth | service_account (`mcp-web-app-wordpress`) | read-only | 3 | plugin | `web-app-hermes`, `web-app-openclaw` | โ€” | diff --git a/tests/external/update/docker/test_image_pin_policy.py b/tests/external/update/docker/test_image_pin_policy.py index e9c572aa60..be4e27c3fa 100644 --- a/tests/external/update/docker/test_image_pin_policy.py +++ b/tests/external/update/docker/test_image_pin_policy.py @@ -72,8 +72,7 @@ class TestImagePinPolicy(unittest.TestCase): f" roles/{ref.role}/{ROLE_FILE_META_SERVICES}: {ref.service} pins " f"{ref.version!r}, which is neither a version nor a digest" for ref in refs - if pin_class(ref.version) == REF - and not _declared(ref, RULE_BY_CLASS[REF]) + if pin_class(ref.version) == REF and not _declared(ref, RULE_BY_CLASS[REF]) ] if offenders: diff --git a/tests/unit/python/plugins/filter/test_redirect_sources.py b/tests/unit/python/plugins/filter/test_redirect_sources.py index 86149503b1..d0aaa7d712 100644 --- a/tests/unit/python/plugins/filter/test_redirect_sources.py +++ b/tests/unit/python/plugins/filter/test_redirect_sources.py @@ -47,7 +47,9 @@ class TestRedirectSources(unittest.TestCase): self.assertEqual(redirect_sources(" unpkg.com ")[0], "https://unpkg.com/") def test_the_host_is_not_otherwise_rewritten(self): - self.assertEqual(redirect_sources("CDN.Example.COM")[1], "http://CDN.Example.COM/") + self.assertEqual( + redirect_sources("CDN.Example.COM")[1], "http://CDN.Example.COM/" + ) def test_a_subdomain_host_keeps_every_label(self): self.assertEqual( diff --git a/tests/unit/python/utils/docker/test_image_pin.py b/tests/unit/python/utils/docker/test_image_pin.py index a8737c8157..bf95b2650d 100644 --- a/tests/unit/python/utils/docker/test_image_pin.py +++ b/tests/unit/python/utils/docker/test_image_pin.py @@ -57,7 +57,9 @@ class TestPullReference(unittest.TestCase): def test_a_tag_is_joined_with_a_colon(self): self.assertEqual(reference_separator("3.0.1"), ":") - self.assertEqual(pull_reference("ghcr.io/owner/app", "3.0.1"), "ghcr.io/owner/app:3.0.1") + self.assertEqual( + pull_reference("ghcr.io/owner/app", "3.0.1"), "ghcr.io/owner/app:3.0.1" + ) def test_is_digest_rejects_a_tag_that_merely_mentions_sha256(self): self.assertFalse(is_digest("v2-sha256:abc")) @@ -73,7 +75,9 @@ class TestSourceKeepsTheDigest(unittest.TestCase): ) def test_image_source_still_tags_a_semver(self): - self.assertEqual(image_source("ghcr.io/owner/app", "3.0.1"), "ghcr.io/owner/app:3.0.1") + self.assertEqual( + image_source("ghcr.io/owner/app", "3.0.1"), "ghcr.io/owner/app:3.0.1" + ) def test_docker_hub_source_keeps_a_digest(self): self.assertEqual( From c4556fe72d495f9f7c17684169127d180c6bda3c Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 21:12:42 +0200 Subject: [PATCH 113/197] fix(cache): key the applications SPOT by the groups its payload reads A service flag may be written as `{{ '' in group_names }}`; 114 meta/services.yml files do exactly that, and it is the only host-dependent variable any of them uses. The rendered payload therefore differs per host while the cache key did not: it was built from applications, users, DOMAIN_PRIMARY and SYSTEM_EMAIL_DOMAIN alone, and in the failing job all five host_vars files were byte-identical. 113113e553 moved the render out of every task on the premise that Ansible forks a fresh worker per task, so the in-process cache is always cold. That holds for workers and not for the strategy's main process, which renders a task's name once for the first host of a batch. Anything cached there is inherited by every worker forked afterwards, and applications.py consults that process cache before the host's own carrier fact, so an inherited entry outranks the correct answer. Pretix#2 of run 35152776835 shows both sides in one task: `skipping: [web-app-pretix-swarm-mgr-01]` against `included: dedicated.yml for wrk-01, wrk-02`, eleven seconds before mgr-01 resolved its database to `pretix_database`. The swarm was running the central `postgres_postgres`, so the wait never converged and the job died after 600 attempts. Variants 0 and 1 pin the flag to a literal and both pass; only variant 2 inherits the Jinja. The groups join the key as a sorted comma-joined string rather than a tuple, because carrier.py stringifies only one side of its key comparison. A non-scalar component would make the carrier silently stop matching and every lookup re-render. domains derives from the applications view and inherits the same dependency, so this closes a second instance that had not surfaced yet. users reads group_names nowhere. No extra rendering results: group_names is constant inside a worker, and the one process that sees several hosts is the one the split is for. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/python/utils/cache/test_base.py | 26 ++++++++++++++++++++-- tests/unit/python/utils/cache/test_data.py | 4 ++-- utils/cache/base.py | 11 ++++++++- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/tests/unit/python/utils/cache/test_base.py b/tests/unit/python/utils/cache/test_base.py index a25a5cbac9..044cd168e9 100644 --- a/tests/unit/python/utils/cache/test_base.py +++ b/tests/unit/python/utils/cache/test_base.py @@ -111,13 +111,35 @@ class TestStableVariablesSignature(unittest.TestCase): def test_empty_variables_collapses_to_canonical_tuple(self): self.assertEqual( base._stable_variables_signature(None), - ("0", "0", "", ""), + ("0", "0", "", "", ""), ) self.assertEqual( base._stable_variables_signature({}), - ("0", "0", "", ""), + ("0", "0", "", "", ""), ) + def test_two_hosts_of_one_play_do_not_share_a_key(self): + shared = {"applications": {"web-app-pretix": {}}, "DOMAIN_PRIMARY": "x.test"} + member = base._stable_variables_signature( + {**shared, "group_names": ["web-app-pretix", "svc-db-postgres"]} + ) + outsider = base._stable_variables_signature( + {**shared, "group_names": ["web-app-pretix"]} + ) + + self.assertNotEqual( + member, + outsider, + "a service flag may read group_names, so one host's payload must " + "never answer for a host whose groups differ", + ) + + def test_group_order_does_not_split_the_key(self): + first = base._stable_variables_signature({"group_names": ["a", "b"]}) + second = base._stable_variables_signature({"group_names": ["b", "a"]}) + + self.assertEqual(first, second) + def test_includes_domain_primary_and_email_domain_strings(self): sig = base._stable_variables_signature( { diff --git a/tests/unit/python/utils/cache/test_data.py b/tests/unit/python/utils/cache/test_data.py index 9744417d79..c8ce41ad14 100644 --- a/tests/unit/python/utils/cache/test_data.py +++ b/tests/unit/python/utils/cache/test_data.py @@ -129,8 +129,8 @@ class TestStableVariablesSignature(unittest.TestCase): _reset_cache_for_tests() def test_empty_variables_returns_sentinel(self): - self.assertEqual(_stable_variables_signature(None), ("0", "0", "", "")) - self.assertEqual(_stable_variables_signature({}), ("0", "0", "", "")) + self.assertEqual(_stable_variables_signature(None), ("0", "0", "", "", "")) + self.assertEqual(_stable_variables_signature({}), ("0", "0", "", "", "")) def test_captures_key_subset(self): sig = _stable_variables_signature( diff --git a/utils/cache/base.py b/utils/cache/base.py index 9986e7fe63..f9c2d23276 100644 --- a/utils/cache/base.py +++ b/utils/cache/base.py @@ -104,15 +104,24 @@ def _stable_variables_signature(variables: Mapping[str, Any] | None) -> tuple: """Build a content-based cache signature from the subset of `variables` that influences the merged applications/users payload. + `group_names` belongs in the key because a service flag may be written as + `{{ '' in group_names }}`, so two hosts of one play render different + payloads from identical host_vars. Ansible forks its workers, so a payload + rendered in the strategy's main process for one host is inherited by every + later worker; without this component that payload answers for hosts whose + groups say otherwise. + See `_fingerprint_mapping` for why id()-only keys don't work reliably. """ if not variables: - return ("0", "0", "", "") + return ("0", "0", "", "", "") + groups = variables.get("group_names") return ( _fingerprint_mapping(variables.get("applications")), _fingerprint_mapping(variables.get("users")), str(variables.get("DOMAIN_PRIMARY") or ""), str(variables.get("SYSTEM_EMAIL_DOMAIN") or ""), + ",".join(sorted(str(group) for group in groups)) if groups else "", ) From 6e94e5b213a215a1b324eed6b548a00b3dee3bef Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 21:16:27 +0200 Subject: [PATCH 114/197] docs(cache): draw the SPOT lookup path and the three ways it lies The applications SPOT is read through four places that each look authoritative on their own: the process-global dict, the per-host carrier fact, the templar render, and the key that decides between them. Which one answers a given lookup is only visible when the order is written down, and the order is what c4556fe72 turned out to depend on. The diagram is scoped to that path and carries the three trip-wires on the edges where they fire rather than in a list underneath: the process cache is consulted before the host's own fact, the strategy's main process renders a task name for the first host of a batch and every worker forked afterwards inherits it, and carrier.py:68 stringifies only one side of its key comparison, so a non-scalar key component silently stops matching. Node labels avoid Jinja braces because GitHub's Mermaid renderer treats them as node-shape syntax and drops the block. Co-Authored-By: Claude Opus 5 (1M context) --- utils/cache/README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/utils/cache/README.md b/utils/cache/README.md index ec5607b0a6..a4d0080e34 100644 --- a/utils/cache/README.md +++ b/utils/cache/README.md @@ -55,3 +55,34 @@ from utils.cache.domains import get_merged_domains # Tests from utils.cache import _reset_cache_for_tests ``` + +## Schema ๐Ÿ—บ๏ธ + +```mermaid +flowchart TD + decl["meta/services.yml
a flag reading group_names
114 role files"] + ctor["01_constructor.yml:191
applications_carrier"] + fact["_INFINITO_APPLICATIONS_RENDERED
per-host fact"] + lookup["lookup('database', ...)
and every applications consumer"] + key["base.py:103
_stable_variables_signature"] + proc["_MERGED_APPLICATIONS_CACHE
process-global dict"] + render["base.py:198
_render_with_templar"] + consumer["database.py:121
central_enabled = shared"] + name["postgres_postgres
vs pretix_database"] + + decl -->|"read by"| render + ctor -->|"parks"| fact + lookup -->|"computes"| key + key -->|"1. looked up in"| proc + proc -->|"2. miss: falls back to"| fact + fact -->|"3. miss: renders"| render + render -->|"substitutes group_names,
then stores under key"| proc + render --> consumer + consumer -->|"selects"| name + + proc -.->|"TRIP-WIRE: read BEFORE the
host's own fact, so an inherited
entry outranks it"| fact + render -.->|"TRIP-WIRE: main process renders
task names for one host; forked
workers inherit that entry"| proc + key -.->|"TRIP-WIRE: carrier.py:68 stringifies
only one side, so a non-scalar key
part silently never matches again"| fact +``` + +The key covers `group_names`. `domains.py` derives from the applications view and shares that dependency; `users` reads `group_names` nowhere. From 152c7b16aea037f4652644fbc297fbac71691491 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 22:04:01 +0200 Subject: [PATCH 115/197] fix(socialhome): declare the inline permission the app's own page needs The served document runs an inline script at line 38. It is not ours: in the failing variant the Lua body filter renders an empty snippet list, so nothing this repository injects can account for it. Until now the only token that admitted that script came from csp_filters.py, which grants 'unsafe-inline' to script-src-attr and script-src-elem whenever services.logout is enabled. socialhome derives that flag from 'web-svc-logout' in group_names, so the app's own page worked or broke depending on whether an unrelated service shared the round. Measured in run 35152776835. Job 105253846799 reports `Blocked URI: inline`, `Directive: script-src-elem`, `Source: /:38:0`. The two rendered headers differ in exactly that token: variant 0 emits `script-src-elem 'self' 'unsafe-inline' ...`, variant 1 emits `script-src-elem 'self' ...`. csp-checker exits with the violation count, so this single violation failed the hlth-csp unit and sys-service-terminator took the job down after 25 attempts. Rendering build_csp_header against HEAD and against this change confirms the delta is bounded: with logout off the directive gains the token, with logout on nothing moves. script-src-attr stays undeclared because no artifact reports a violation there. An inline hash was considered and dropped. No role in this repository declares meta/csp.yml hashes, the hash would have to byte-match the served document, and csp_filters.py discards every registered hash whenever 'unsafe-inline' is present, which is the state this app is in as soon as logout is deployed. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-socialhome/meta/csp.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/web-app-socialhome/meta/csp.yml b/roles/web-app-socialhome/meta/csp.yml index 6258ff8fbc..5896abc89e 100644 --- a/roles/web-app-socialhome/meta/csp.yml +++ b/roles/web-app-socialhome/meta/csp.yml @@ -8,3 +8,6 @@ whitelist: - https://fonts.googleapis.com # nocheck: external-cdn,url font-src: - https://fonts.gstatic.com # nocheck: external-cdn,url +flags: + script-src-elem: + unsafe-inline: true # nocheck: csp-unsafe Reason: the served document carries an inline script of its own at line 38, which csp-checker reports and hlth-csp then fails on From 1941f0a68902c895b62659cf39e5f9537866a40d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 22:22:03 +0200 Subject: [PATCH 116/197] docs(csp): draw how an application's policy is assembled and where it lies One directive is fed from four places that each look authoritative on their own: the role's own flags, the whitelist, the hashes the front injectors register, and one branch that reads a service flag. Which of them ends up in the header is only visible when the order is written down, and that order is what the socialhome failure turned out to depend on. The diagram is scoped to the header build and carries the three trip-wires on the edges where they fire. The logout branch adds 'unsafe-inline' and thereby closes the hash gate directly below it, so every registered hash is discarded, including the one the logout injector computes for its own snippet. The flag it reads derives from group_names, so the script policy follows the round rather than the application. And the branch is absent from the smart-defaults docstring, which names logout only for frame-ancestors. Counted at this commit: 83 roles declare a logout service. 59 declare their own script flag, 16 pin logout to false and have never had the grant, and 8 rely on it with nothing of their own. That last group is what a reader needs the diagram for. Node labels carry file and line so a policy that surprises someone can be traced in one jump instead of a read through 460 lines. Co-Authored-By: Claude Opus 5 (1M context) --- plugins/filter/README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/plugins/filter/README.md b/plugins/filter/README.md index 8ad96bd0ab..7312a1d39a 100644 --- a/plugins/filter/README.md +++ b/plugins/filter/README.md @@ -21,6 +21,39 @@ This directory contains custom **Ansible filter plugins** used within the Infini - If you need to **load data from an external source** (e.g., file, environment, API), use a lookup plugin instead. - If your logic requires **access to inventory, facts, or host-level information** that is not passed as a parameter. +## Schema ๐Ÿ—บ๏ธ + +How `csp_filters.py` assembles one application's Content-Security-Policy. Scope is the header build, not the other filters in this directory. + +```mermaid +flowchart TD + decl["meta/csp.yml
whitelist, flags, hashes"] + svc["meta/services.yml
services.logout.enabled"] + flags["csp_filters.py:136
get_csp_flags"] + grant["csp_filters.py:354
logout grant"] + wl["csp_filters.py:123
get_csp_whitelist"] + inj["sys-front-inj-dashboard,
-javascript, -matomo, -logout
add_csp_hash"] + gate["csp_filters.py:365
hash gate"] + build["csp_filters.py:203
build_csp_header"] + header["set $csp
nginx vhost"] + + decl -->|"1. flags read by"| flags + flags -->|"2. tokens"| build + svc -->|"3. read by"| grant + grant -->|"4. adds 'unsafe-inline'
to script-src-elem and -attr"| build + decl -->|"5. whitelist read by"| wl + wl -->|"6. tokens"| build + inj -->|"7. snippets"| gate + gate -->|"8. hashes, only when
'unsafe-inline' is absent"| build + build -->|"9. joined string"| header + + grant -.->|"TRIP-WIRE: the token it adds
closes the gate below, so every
registered hash is discarded"| gate + svc -.->|"TRIP-WIRE: the flag derives from
group_names, so the script policy
follows the round, not the app"| grant + grant -.->|"TRIP-WIRE: absent from the
smart-defaults docstring, which
names logout only for frame-ancestors"| build +``` + +Styles carry `'unsafe-inline'` by default; scripts do not. The logout grant is the only path that puts it into a script directive. + ## Further Reading - [Ansible Filter Plugins Documentation](https://docs.ansible.com/ansible/latest/plugins/filter.html) From ee582762b06f3edca41e0b6a54712380f5697b27 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 17 Sep 2026 23:46:27 +0200 Subject: [PATCH 117/197] feat(lmstudio): fetch the model ourselves instead of trusting one client shot The preload is the only artifact this repository pulls raw from the public internet. Everything else, including the LM Studio image itself, arrives through the mirror. `lms get` gets exactly one attempt under async, cannot resume, and wedges its own daemon when a transfer dies, so one dropped socket costs a whole job. XWiki#2 of run 35152776835 died 28 seconds into the transfer at 3 MB/s and rising, with 3600 seconds of budget left. The model is now fetched on the hosting node with get_url_retry, which gives 13 attempts and verifies a pinned sha256, and handed to the container with `lms import`. Both steps are best effort: when the warm-up fails, `lms get` runs exactly as before, so the change adds a faster path without removing the old one. The pattern is web-app-mediawiki's extension install, down to the `container cp` in the script. Verified against the binary the role actually deploys, extracted from lmstudio/llmster-preview:cpu and built 2025-08-28, not against upstream HEAD: `lms import` takes ``, `-y, --yes` and `--user-repo `; the destination is join(modelsFolderPath, user, repo, basename(path)) under join(lmstudioHome, "models"); the image ships no settings.json, so downloadsFolder cannot move that target. The already-present branch of `lms get` carries no process.exit, so it returns 0 and the task's existing `until: rc == 0` accepts it. The pin carries file and sha256 next to the source URL, and the digest is quoted because an all-digit hash would otherwise parse as an integer and fail the shape check for the wrong reason. tests/external guards the pin against what HuggingFace publishes, comparing digests through the API without transferring the file, and reports a network error as unverified rather than failing a good pin on a firewalled runner. Two limits worth stating. `lms get` still queries HuggingFace before it reaches the download step, so this removes 531 MB of exposure rather than the network dependency. And the tolerated substring 'already downloaded' in the task's until, failed_when and reap list matches no output this version emits, which says "You already have this model."; that branch is dead today and is left untouched here because the return code carries the contract. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/16_ai.yml | 2 + roles/svc-ai-lmstudio/files/shell/import.sh | 33 +++ .../svc-ai-lmstudio/tasks/01_manager_ops.yml | 6 + roles/svc-ai-lmstudio/tasks/utils/seed.yml | 33 +++ roles/svc-ai-lmstudio/vars/main.yml | 4 + .../test_ai_local_models_downloadable.py | 202 ++++++++++++++++++ 6 files changed, 280 insertions(+) create mode 100644 roles/svc-ai-lmstudio/files/shell/import.sh create mode 100644 roles/svc-ai-lmstudio/tasks/utils/seed.yml create mode 100644 tests/external/roles/test_ai_local_models_downloadable.py diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index 5a5c5f2870..e5c6ee6df1 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -25,6 +25,8 @@ AI_LOCAL_MODELS: - alias: qwen2.5:0.5b name: qwen2.5-0.5b-instruct source: https://huggingface.co/lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF + file: Qwen2.5-0.5B-Instruct-Q8_0.gguf + sha256: "1d96e30253a76970864323987e3e77eeaf170312952fd61d18d343c25cfd6e52" AI_REMOTE_ALIASES: >- {{ ['openai/gpt-4o-mini'] * (lookup('api_enabled', 'openai') | int) diff --git a/roles/svc-ai-lmstudio/files/shell/import.sh b/roles/svc-ai-lmstudio/files/shell/import.sh new file mode 100644 index 0000000000..98667d5d02 --- /dev/null +++ b/roles/svc-ai-lmstudio/files/shell/import.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Hand a pre-fetched GGUF to LM Studio, idempotent. +# Runs ON THE HOSTING NODE (delegated); targets the local container via +# `container exec` / `container cp`. +# +# Required env, supplied by the calling Ansible task: +# LMSTUDIO_LOCAL_CID resolved container id, local to this node +# MODEL_FILE node-side path to the fetched .gguf +# MODEL_USER_REPO HuggingFace "user/repo" the file belongs to +# MODELS_DIR models folder inside the container, under the +# volume target declared in meta/volumes.yml +set -euo pipefail + +ADDRESS="${LMSTUDIO_LOCAL_CID:?LMSTUDIO_LOCAL_CID env var (container id) required}" +BASE="$(basename "${MODEL_FILE}")" +TARGET="${MODELS_DIR}/${MODEL_USER_REPO}/${BASE}" + +if container exec "$ADDRESS" test -f "${TARGET}"; then + echo "PRESENT:${BASE}" + exit 0 +fi + +if [ ! -f "${MODEL_FILE}" ]; then + echo "ABSENT:${BASE}" + exit 0 +fi + +# nocheck: container-cp - MODEL_FILE is fetched on the node this script runs on +container cp "${MODEL_FILE}" "${ADDRESS}:/tmp/${BASE}" + +container exec "$ADDRESS" lms import "/tmp/${BASE}" --yes --user-repo "${MODEL_USER_REPO}" +echo "IMPORTED:${BASE}" diff --git a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml index b588c180f2..6cffa1413f 100644 --- a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml +++ b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml @@ -6,6 +6,12 @@ resolve_output_prefix: LMSTUDIO resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" +- name: "๐ŸŒฑ Seed the models from the node cache before preloading" + ansible.builtin.include_tasks: + file: utils/seed.yml + apply: + delegate_to: "{{ LMSTUDIO_HOST_NODE }}" + - name: "โฌ‡๏ธ Preload the LM Studio models" block: - name: "โฌ‡๏ธ Preload the LM Studio models (first pass)" diff --git a/roles/svc-ai-lmstudio/tasks/utils/seed.yml b/roles/svc-ai-lmstudio/tasks/utils/seed.yml new file mode 100644 index 0000000000..ec28cea89e --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/seed.yml @@ -0,0 +1,33 @@ +--- +- name: "๐Ÿ“ Create the model cache directory on the hosting node" + ansible.builtin.file: + path: "{{ LMSTUDIO_MODEL_CACHE_DIR }}" + state: directory + mode: "0755" + +- name: "โฌ‡๏ธ Warm the model cache on the node, best effort" + get_url_retry: + url: "{{ model.source }}/resolve/main/{{ model.file }}" + dest: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ model.file }}" + checksum: "sha256:{{ model.sha256 }}" + mode: "0644" + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + changed_when: false + failed_when: false + +- name: "๐Ÿ“‹ Hand a warmed model to LM Studio, best effort" + ansible.builtin.script: shell/import.sh + environment: + LMSTUDIO_LOCAL_CID: "{{ LMSTUDIO_LOCAL_CID }}" + MODEL_FILE: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ model.file }}" + MODEL_USER_REPO: "{{ model.source | regex_replace('^https://huggingface\\.co/', '') }}" + MODELS_DIR: "{{ LMSTUDIO_MODELS_DIR }}" + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + changed_when: false + failed_when: false diff --git a/roles/svc-ai-lmstudio/vars/main.yml b/roles/svc-ai-lmstudio/vars/main.yml index ce7a2585e4..1dae1fe5e8 100644 --- a/roles/svc-ai-lmstudio/vars/main.yml +++ b/roles/svc-ai-lmstudio/vars/main.yml @@ -6,3 +6,7 @@ LMSTUDIO_CONTAINER: "{{ lookup('config', application_id, 'services.lmstudio.name LMSTUDIO_PORT: "{{ lookup('config', application_id, 'services.lmstudio.ports.internal.http') }}" LMSTUDIO_PRELOAD_MODELS: "{{ lookup('config', application_id, 'services.lmstudio.preload_models') }}" LMSTUDIO_PRELOAD_ASYNC_TIME: "{{ lookup('config', application_id, 'services.lmstudio.preload_async_time') }}" +LMSTUDIO_CONFIG_DIR: "{{ lookup('container', application_id, 'directories.config') }}" +LMSTUDIO_MODEL_CACHE_DIR: "{{ [ LMSTUDIO_CONFIG_DIR, 'models' ] | path_join }}" +LMSTUDIO_HOME_DIR: "{{ lookup('volume', application_id, 'lmstudio_data').target }}" +LMSTUDIO_MODELS_DIR: "{{ [ LMSTUDIO_HOME_DIR, 'models' ] | path_join }}" diff --git a/tests/external/roles/test_ai_local_models_downloadable.py b/tests/external/roles/test_ai_local_models_downloadable.py new file mode 100644 index 0000000000..cd6785ed74 --- /dev/null +++ b/tests/external/roles/test_ai_local_models_downloadable.py @@ -0,0 +1,202 @@ +"""Every declared local AI model still resolves to a file upstream serves. + +``group_vars/all/16_ai.yml`` pins each model by repository, file name and +sha256 so a deploy fetches it with a checksum instead of trusting whatever a +search term resolves to. A pin that upstream no longer matches becomes a deploy +that transfers half a gigabyte on every host and then fails on the digest, with +nothing in the failure naming the declaration as the cause. + +The declaration's own shape is a repository fact and is asserted. Whether the +file is still served is a network fact, so a timeout or a refused connection is +reported as unverified rather than failing, the way the sibling external checks +do: a firewalled runner must not turn a good pin red. An answer that contradicts +the pin is a defect in the repository and does fail. +""" + +from __future__ import annotations + +import concurrent.futures +import re +import unittest +from urllib.parse import urlsplit + +import requests + +from utils.annotations.message import warning +from utils.cache.yaml import load_yaml_any + +from . import PROJECT_ROOT + +_AI_VARS_FILE = "group_vars/all/16_ai.yml" +_HUGGINGFACE_PREFIX = "https://huggingface.co/" +_USER_AGENT = "infinito-nexus-model-pin-check" +_TIMEOUT_SECONDS = 20 +_MAX_WORKERS = 4 +_SHA256 = re.compile(r"^[0-9a-f]{64}$") + + +def _models() -> list[dict]: + data = load_yaml_any(str(PROJECT_ROOT / _AI_VARS_FILE)) or {} + entries = data.get("AI_LOCAL_MODELS") if isinstance(data, dict) else None + return [entry for entry in (entries or []) if isinstance(entry, dict)] + + +def _repo_path(source: str) -> str: + """Return ``user/repo`` for a HuggingFace source, else the empty string.""" + if not source.startswith(_HUGGINGFACE_PREFIX): + return "" + return source[len(_HUGGINGFACE_PREFIX) :].strip("/") + + +def _download_url(entry: dict) -> str: + return f"{entry['source'].rstrip('/')}/resolve/main/{entry['file']}" + + +def _upstream_digests(repo_path: str) -> tuple[str, object]: + """Return (kind, payload) with kind one of ok, warn, unverified. + + On ``ok`` the payload maps each published file name to its sha256, which is + what the pin is compared against without transferring the file itself. + """ + url = f"{_HUGGINGFACE_PREFIX}api/models/{repo_path}" + try: + response = requests.get( + url, + params={"blobs": "true"}, + headers={"User-Agent": _USER_AGENT}, + timeout=_TIMEOUT_SECONDS, + ) + except requests.Timeout as exc: + return "unverified", f"Timeout: {exc}" + except requests.RequestException as exc: + return "unverified", f"{type(exc).__name__}: {exc}" + + if response.status_code != 200: + return "warn", f"HTTP {response.status_code}" + try: + payload = response.json() + except ValueError as exc: + return "unverified", f"unparsable answer: {exc}" + + digests = {} + for sibling in payload.get("siblings") or []: + if not isinstance(sibling, dict): + continue + name = sibling.get("rfilename") + digest = (sibling.get("lfs") or {}).get("sha256") + if isinstance(name, str) and isinstance(digest, str): + digests[name] = digest + return "ok", digests + + +def _reachable(url: str) -> tuple[str, str]: + """Return (kind, detail); a redirect to the CDN counts as reachable.""" + try: + response = requests.get( + url, + allow_redirects=True, + headers={"User-Agent": _USER_AGENT, "Range": "bytes=0-0"}, + stream=True, + timeout=_TIMEOUT_SECONDS, + ) + try: + status = response.status_code + finally: + response.close() + except requests.Timeout as exc: + return "unverified", f"Timeout: {exc}" + except requests.RequestException as exc: + return "unverified", f"{type(exc).__name__}: {exc}" + + if status < 400: + return "ok", f"HTTP {status}" + return "warn", f"HTTP {status}" + + +class TestAiLocalModelsDownloadable(unittest.TestCase): + def test_every_declared_model_is_pinned(self) -> None: + models = _models() + self.assertTrue(models, f"{_AI_VARS_FILE} declares no AI_LOCAL_MODELS") + + unpinned = [] + for entry in models: + alias = str(entry.get("alias") or "") + source = entry.get("source") + file_name = entry.get("file") + digest = entry.get("sha256") + if not isinstance(source, str) or not urlsplit(source).scheme: + unpinned.append(f"{alias}: 'source' is not an absolute URL") + if not isinstance(file_name, str) or not file_name: + unpinned.append(f"{alias}: 'file' is missing") + if not isinstance(digest, str) or not _SHA256.match(digest): + unpinned.append(f"{alias}: 'sha256' is not 64 hex characters") + + self.assertFalse( + unpinned, + f"{len(unpinned)} model declaration(s) in {_AI_VARS_FILE} cannot be " + f"fetched with a checksum. Each entry needs 'source', 'file' and " + f"'sha256':\n" + "\n".join(f" {line}" for line in unpinned), + ) + + def test_every_pin_matches_what_upstream_serves(self) -> None: + models = [ + entry + for entry in _models() + if isinstance(entry.get("source"), str) + and isinstance(entry.get("file"), str) + and isinstance(entry.get("sha256"), str) + ] + self.assertTrue(models, f"{_AI_VARS_FILE} declares no pinned model") + + repo_paths = [_repo_path(entry["source"]) for entry in models] + with concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_WORKERS) as pool: + api = list(pool.map(_upstream_digests, repo_paths)) + reach = list(pool.map(_reachable, [_download_url(e) for e in models])) + + mismatched = [] + for entry, repo_path, (kind, payload), (rkind, rdetail) in zip( + models, repo_paths, api, reach, strict=True + ): + alias = entry["alias"] + url = _download_url(entry) + if rkind != "ok": + warning( + f"{alias}: {url} answered {rdetail}" + if rkind == "warn" + else f"{alias}: {url} was not checked ({rdetail})", + title="Model unreachable" + if rkind == "warn" + else "Model unverified", + file=_AI_VARS_FILE, + ) + if not repo_path: + continue + if kind != "ok": + warning( + f"{alias}: {repo_path} was not checked ({payload})", + title="Model pin unverified", + file=_AI_VARS_FILE, + ) + continue + published = payload.get(entry["file"]) + if published is None: + mismatched.append( + f"{alias}: {repo_path} no longer publishes {entry['file']}" + ) + elif published != entry["sha256"]: + mismatched.append( + f"{alias}: {entry['file']} is {published} upstream, " + f"pinned as {entry['sha256']}" + ) + + self.assertFalse( + mismatched, + f"{len(mismatched)} model pin(s) in {_AI_VARS_FILE} no longer match " + f"what upstream publishes. Update 'file' and 'sha256' to the value " + f"the repository serves today:\n" + + "\n".join(f" {line}" for line in mismatched), + ) + + +if __name__ == "__main__": # pragma: no cover + unittest.main() From b8c651be3bcd741a5011a6ed0beea546534b9543 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 00:04:21 +0200 Subject: [PATCH 118/197] fix(hlth-csp): take the csp-checker build where the 204 handling fires 3.0.2 never produced an image. Its release build failed the E2E suite on the 204 fixture itself: the handling listened for Network.responseReceived, and a 204 is never committed, so Chromium emits no such event and the checker kept reporting the host as unreachable. The pin therefore addressed a tag that the registry answers with 404, which the image-reachability check has been saying since the bump landed. csp-checker d827de1 reads the status from Network.responseReceivedExtraInfo, the event that carries an uncommitted navigation's status, correlated with the Document request id. Reproduced and fixed in one E2E run, with the download fixture still failing so the change distinguishes a 204 rather than blessing every aborted navigation. A new number rather than a rebuilt 3.0.2, because the mirror copies by tag: an overwritten tag turns every copy that already saw 3.0.2 into a silent lie, and the deploys pull through that mirror. Until the release publishes, the image does not exist and the reachability check answers 404 for 3.0.3 the way it did for 3.0.2. That clears when the release runs, not by a change here. Co-Authored-By: Claude Opus 5 (1M context) --- roles/sys-ctl-hlth-csp/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sys-ctl-hlth-csp/meta/services.yml b/roles/sys-ctl-hlth-csp/meta/services.yml index 90cae220c6..efa7695b5b 100644 --- a/roles/sys-ctl-hlth-csp/meta/services.yml +++ b/roles/sys-ctl-hlth-csp/meta/services.yml @@ -8,4 +8,4 @@ csp: categories: [web] csp-checker: image: ghcr.io/kevinveenbirkenbach/csp-checker - version: 3.0.2 + version: 3.0.3 From a534d5d46eb7d2d60f2820147f939edd9b2370f1 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 01:00:02 +0200 Subject: [PATCH 119/197] fix(hlth-csp): take the csp-checker build whose 204 no longer races 3.0.3 has a tag and no image. Its release build failed the same 204 fixture 3.0.2 failed on, because the handling read the status the moment page.goto rejected while the listener sat on a second CDP session, and the protocol orders messages within a session rather than between two of them. The status now gets awaited, so the verdict no longer depends on which session delivers first, and the page close that the wait pushes into the navigation's commit window is bounded rather than awaited forever. A third number rather than a rebuilt 3.0.3, for the reason 3.0.3 itself got one: the mirror copies by tag, so an overwritten tag turns every copy that has already seen it into a silent lie, and the deploys pull through that mirror. The reachability check answers 404 for 3.0.4 until the release publishes, the way it did for the two before it. That clears when the release runs. Worth stating plainly: the runner's failure was never reproduced locally. The ordering hole is closed by construction and two full suites are green, but whether it was the cause is decided by the release build, not here. If that build fails on the 204 fixture again, this pin is worth as little as the last one and the job log is what to read next. Co-Authored-By: Claude Opus 5 (1M context) --- roles/sys-ctl-hlth-csp/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/sys-ctl-hlth-csp/meta/services.yml b/roles/sys-ctl-hlth-csp/meta/services.yml index efa7695b5b..a6e9695221 100644 --- a/roles/sys-ctl-hlth-csp/meta/services.yml +++ b/roles/sys-ctl-hlth-csp/meta/services.yml @@ -8,4 +8,4 @@ csp: categories: [web] csp-checker: image: ghcr.io/kevinveenbirkenbach/csp-checker - version: 3.0.3 + version: 3.0.4 From 86f7d8c4f5b113e85f8da42d97123836c78db035 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 03:03:37 +0200 Subject: [PATCH 120/197] fix(install): keep the collection check inside the slice that runs it The image runs this module from a layer that holds three files of utils: __init__.py, install/__init__.py and install/collections.py, the slice ef45d69dc3 cut so the source tree cannot invalidate the collection install. 67f5398985 established that the module therefore stays on the standard library, and d4ae5cf9f0 broke it 15 minutes later by routing the reads through utils.cache.files to satisfy the cache-read lint. The import raises ModuleNotFoundError: No module named 'utils.cache' in that layer, which no import inside a full checkout can show, because this repository installs utils into site-packages. Nothing turned red. The caller reads only the exit code, and the module's contract is exit 1 with the missing collections on stdout, so a crash - exit 1, nothing on stdout - is indistinguishable from work to do. Every image build since has reinstalled every collection with --force-with-deps, which is the cost the check exists to avoid. The deploy job the traceback was found in finished green. The reads go back to Path.read_text with a cache-read marker each. The cache was never earning its place here: read_text is an lru_cache keyed on the path, and one run reads the requirements file once and one MANIFEST.json per collection, eight distinct paths, none of them twice. Copying utils/cache into the slice instead would work - the closure is three stdlib-only files - but it binds a package with 51 commits in the last twelve months to the layer that installs ansible, in front of a cache with no possible hit. The caller now exits 3 when a non-zero status arrives with an empty stdout, a state the contract cannot produce, so the next crash stops the install instead of quietly reinstalling everything. all.sh runs under set -e and touches the stamp only after ansible.sh returns, so the failure also leaves the stamp stale rather than recording an install that did not happen. The probe that would have caught this builds the slice in a tempdir from the Dockerfile's own COPY lines and runs the module inside it. It has to pass python -S: without it the interpreter resolves every submodule from the installed checkout however the slice is built, and the test passes on a module that cannot import in the image. Verified by injecting the import back into the tempdir copy, which reproduces the traceback verbatim. Delta measured by loading the committed module and this one side by side: identical on both requirements files, against the real collections directory and an empty one, on a missing file, on non-UTF-8 input and on a corrupt MANIFEST.json. The one behavioural difference is that a file changed within a process is now read fresh rather than from the memo. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/install/ansible.sh | 5 ++ .../python/utils/install/test_collections.py | 81 +++++++++++++++++++ utils/install/collections.py | 14 ++-- 3 files changed, 95 insertions(+), 5 deletions(-) diff --git a/scripts/install/ansible.sh b/scripts/install/ansible.sh index 6f5ce14ed8..8aa2d5b0e1 100755 --- a/scripts/install/ansible.sh +++ b/scripts/install/ansible.sh @@ -42,6 +42,11 @@ if MISSING="$("${PYTHON}" -m utils.install.collections "${GALAXY_REQ}" "${ANSIBL echo "๐ŸŽ‰ All collections are ready" exit 0 fi +if [ -z "${MISSING}" ]; then + echo "โŒ The collection check exited non-zero without naming a collection," \ + "so it crashed rather than finding work; see its traceback above" >&2 + exit 3 +fi echo "โ†’ Not satisfied yet: ${MISSING}" installed=0 diff --git a/tests/unit/python/utils/install/test_collections.py b/tests/unit/python/utils/install/test_collections.py index 7f552aa32e..8ff53e96cb 100644 --- a/tests/unit/python/utils/install/test_collections.py +++ b/tests/unit/python/utils/install/test_collections.py @@ -1,4 +1,6 @@ import json +import re +import shutil import subprocess import sys import unittest @@ -6,9 +8,47 @@ from pathlib import Path from tempfile import TemporaryDirectory from utils import PROJECT_ROOT +from utils.cache.files import read_text from utils.cache.yaml import load_yaml_any from utils.install.collections import declared_pins, unsatisfied +_DOCKERFILE = PROJECT_ROOT / "Dockerfile" +_COPY = re.compile(r"^COPY\s+(?P\S+)\s+\$\{INFINITO_SRC_DIR\}/(?P\S+)$") + + +def _materialise_image_slice(destination: Path) -> None: + """Reproduce what the image holds when it first runs the collection check. + + The Dockerfile copies a handful of paths, runs ``scripts/install/ansible.sh`` + against them, and only afterwards copies the rest of the tree. Anything + ``utils.install.collections`` imports beyond that slice is absent in the + layer that executes it, which no import inside a full checkout can show. + The list is read from the Dockerfile rather than restated here, so a COPY + added or dropped moves this probe with it. + + The caller must run the slice under ``python -S``. This repository installs + ``utils`` into site-packages, so without it the interpreter resolves every + submodule from the checkout however the slice is built, and the probe + passes on a module that cannot import inside the image. + + Args: + destination: an empty directory to build the slice in. + """ + for line in read_text(str(_DOCKERFILE)).splitlines(): + if line.startswith("COPY . "): + return + match = _COPY.match(line.strip()) + if not match: + continue + source = PROJECT_ROOT / match.group("source") + target = destination / match.group("target") + target.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, target, dirs_exist_ok=True) + else: + shutil.copy2(source, target) + + _REQUIREMENTS = ( PROJECT_ROOT / "requirements" / "requirements.galaxy.yml", PROJECT_ROOT / "requirements" / "requirements.git.yml", @@ -89,6 +129,47 @@ class TestAgainstPyYAML(unittest.TestCase): self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("community.general", result.stdout) + def test_the_checker_runs_on_the_slice_the_dockerfile_copies(self): + tmp = TemporaryDirectory() + self.addCleanup(tmp.cleanup) + slice_root = Path(tmp.name) + _materialise_image_slice(slice_root) + requirements = slice_root / "requirements" / "requirements.galaxy.yml" + self.assertTrue( + requirements.is_file(), + "the Dockerfile stopped copying requirements/ before the ansible " + "install layer; this probe needs it to have something to check", + ) + + result = subprocess.run( + [ + sys.executable, + "-S", + "-m", + "utils.install.collections", + str(requirements), + str(slice_root / "collections"), + ], + cwd=str(slice_root), + capture_output=True, + text=True, + check=False, + ) + + self.assertNotIn( + "ModuleNotFoundError", + result.stderr, + "utils.install.collections reached for a module the image layer " + "that runs it does not hold; keep it on the standard library", + ) + self.assertEqual( + result.returncode, + 1, + f"an empty collections dir satisfies nothing, so the checker owes " + f"exit 1 and a list; stderr: {result.stderr}", + ) + self.assertIn("community.general", result.stdout) + class TestDeclaredPins(unittest.TestCase): def test_a_quoted_version_is_read_without_its_quotes(self): diff --git a/utils/install/collections.py b/utils/install/collections.py index 08223a6616..60c2031eb8 100644 --- a/utils/install/collections.py +++ b/utils/install/collections.py @@ -1,7 +1,11 @@ """Compare the pinned Galaxy requirements against what is on disk. Imported by the lint bootstrap, which runs on a bare interpreter before any -dependency is installed, so this module stays on the standard library. The +dependency is installed, so this module stays on the standard library. It also +stays inside its own package: the Dockerfile copies it into the image beside +``utils/__init__.py`` and ``utils/install/__init__.py`` and nothing else, so +importing any other ``utils`` submodule raises ``ModuleNotFoundError`` in the +layer that runs it. The requirements files it reads are this repository's own and hold one shape: a ``collections:`` list of ``- name:`` entries with an optional ``version:``. A line that does not match that shape leaves the entry unpinned, which the @@ -15,8 +19,6 @@ import re import sys from pathlib import Path -from utils.cache.files import read_text - _NAME = re.compile(r"^\s*-\s+name:\s*(?P[A-Za-z0-9_.]+)\s*(?:#.*)?$") _VERSION = re.compile( r"^\s*version:\s*(?P[\"']?)(?P[^\s#\"']+)(?P=quote)\s*(?:#.*)?$" @@ -31,9 +33,10 @@ def declared_pins(requirements_file: Path) -> list[tuple[str, str | None]]: """ pins: list[tuple[str, str | None]] = [] try: - lines = read_text(str(requirements_file)).splitlines() + raw = requirements_file.read_text(encoding="utf-8") # nocheck: cache-read except OSError: return pins + lines = raw.splitlines() for line in lines: name_match = _NAME.match(line) @@ -59,7 +62,8 @@ def _installed_version(collections_dir: Path, namespace: str, name: str) -> str collections_dir / "ansible_collections" / namespace / name / "MANIFEST.json" ) try: - payload = json.loads(read_text(str(manifest))) + raw = manifest.read_text(encoding="utf-8") # nocheck: cache-read + payload = json.loads(raw) except (OSError, ValueError): return None version = payload.get("collection_info", {}).get("version") From c31c62efaf9a8070e0cc72383e4a82b9dd37a43d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 03:18:37 +0200 Subject: [PATCH 121/197] fix(images): install the curl the slim base dropped under the healthcheck 10c5b35e88 moved simpleicons and roulette-wheel from node:26-bookworm to node:26-slim. Both declare healthcheck flavor curl, which renders CMD curl -f , and neither Dockerfile installed anything. Measured with docker create and docker cp: node:26-bookworm carries /usr/bin/curl at 280800 bytes, node:26-slim carries neither curl nor wget. The probe therefore fails from the first beat and the container never reports healthy. sys-svc-compose waits on the database container alone in compose and on every stack task in swarm, so the consequence splits by mode. Under swarm the task dies with non-zero exit (137): dockerexec: unhealthy container, is rescheduled, dies again, and the converge wait fails after 600 attempts. Five jobs across the two runs on a534d5d46 died that way, and none of them on the role they are named after: ERPNext#0, Decidim#0, Hugo#0, Nextcloud#9 and Nextcloud#12 all carry web-app-dashboard in their closure, dashboard enables simpleicons, and the failure reads 'not converged: simpleicons_simpleicons'. All five are swarm and tor, which reads like the tor axis until XWiki#1 is put beside them: swarm, tor, green, and simpleicons is not in its closure. The discriminator is the service, not the axis. Of the seven roles 10c5b35e88 rebased, bridgy-fed and postmarks probe over tcp, and flowise and bookwyrm install curl themselves. These two were the remainder. roulette-wheel had not been deployed yet when this was written, so it is the same defect caught before it was observed. The install goes into the runtime stage of the multi-stage build; the builder stage would have left the runtime image unchanged and nothing would have gone red. The lint reads the binary out of the probe the declaration renders rather than restating a flavor-to-binary table, so a flavor that changes its command moves the check with it, and it locates the flavor line per service so one marker cannot silence a sibling. Its scope stops at slim: nginx:1.31.6-alpine was measured too and ships curl as a real 272 KB binary plus busybox wget, so the four alpine roles that declare those flavors are served by their base, and widening the rule to them needs that measurement per image rather than an assumption from the word alpine. Not built. The image with the new layer has not been produced, and whether a second fault waits behind the healthcheck is decided by the next deploy. The apt path itself is not blind: BookWyrm#1 installs curl the same way on python:3.11-slim and is green in this run. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/testing/suppression.md | 1 + roles/web-app-roulette-wheel/files/Dockerfile | 6 + roles/web-svc-simpleicons/files/Dockerfile | 5 + .../test_healthcheck_binary_on_slim_base.py | 164 ++++++++++++++++++ 4 files changed, 176 insertions(+) create mode 100644 tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py diff --git a/docs/contributing/actions/testing/suppression.md b/docs/contributing/actions/testing/suppression.md index 5e34073123..b0f3939497 100644 --- a/docs/contributing/actions/testing/suppression.md +++ b/docs/contributing/actions/testing/suppression.md @@ -108,6 +108,7 @@ The placement rule is per check. The catalog "Position" column uses these labels | `image-healthcheck` | same or above the service key | [test_image_healthcheck_declared.py](../../../../tests/integration/roles/test_image_healthcheck_declared.py) | Allows a `roles/*/meta/services.yml` entry whose image ships no `HEALTHCHECK` to declare none either. Default path: add `healthcheck:` to the entry, because a container without a probe counts as healthy the moment its process starts โ€” which is how a service that came back empty passes a restore drill. Reserved for containers nothing can probe: a one-shot process that exits, or an image with neither a port nor a shell. | | `csp-unsafe` | same line | [test_csp_unsafe_requires_justification.py](../../../../tests/lint/ansible/roles/meta/test_csp_unsafe_requires_justification.py) | Documents an `unsafe-*: true` flag in `roles/*/meta/csp.yml`. MUST be paired with a `Reason:` on the same line naming what breaks without the relaxation. `unsafe-inline` re-admits the inline execution an XSS needs and `unsafe-eval` re-admits string-to-code, so the flag has to be verified against the running app, not copied from another role. | | `min-storage-headroom` | same line OR comment block above key | [test_min_storage_headroom.py](../../../../tests/external/update/docker/test_min_storage_headroom.py) | Allows a `roles/*/meta/services.yml` `min_storage` to exceed the measured compressed size of its own `image:`/`version:` by more than 1 GB. MUST be paired with a `Reason:` on the same comment line naming what occupies the space. Reserved for services whose data volume genuinely dominates the footprint; the default path is to lower the value, because an over-declared budget makes a host look too small and prunes a cache the deploy did not need to lose. | +| `healthcheck-binary` | same or above | [test_healthcheck_binary_on_slim_base.py](../../../../tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py) | Exempts a `roles/*/meta/services.yml` healthcheck flavor from the requirement that the role's `files/Dockerfile` install the binary the probe runs, when the service sits on a `-slim` image. MUST name where the binary comes from instead. A `-slim` tag carries the runtime and nothing around it, so `curl` is absent and the probe fails from the first beat; under swarm the task is then reaped with exit 137 and every deployment holding the service in its closure fails on the convergence wait, naming the service rather than the missing binary. | | `galaxy-git-parity` | same or above | [test_galaxy_entries_have_git_source.py](../../../../tests/lint/repository/dependencies/test_galaxy_entries_have_git_source.py) | Exempts a `requirements/requirements.galaxy.yml` collection from the requirement to also carry a `type: git` entry in `requirements/requirements.git.yml`. Reserved for collections that publish no usable git remote; without the git entry a `galaxy.ansible.com` outage fails `make install`, because the fallback in `scripts/install/ansible.sh` cannot resolve the collection either. | | `loop-var-shadow` | same or above | [test_loop_var_shadowing.py](../../../../tests/lint/ansible/tasks/test_loop_var_shadowing.py) | Allows a `loop_var` to take the name of a variable declared in `group_vars/**` or in a role's `vars`/`defaults`. Inside the loop that name carries the item, so every lookup, template and nested include reading it sees the item instead, and ansible only warns. Reserved for a loop whose body reads the name as the item on purpose, typically because the item IS that value: state which meaning the body relies on. | | `firewall-fragment-accept` | same or above | [test_firewall_fragment_verdicts.py](../../../../tests/lint/ansible/jinja/test_firewall_fragment_verdicts.py) | Allows a rule in a `roles/*/templates/nftables.conf.j2` fragment to end in an `accept` verdict. Default path: express the intent as a `drop` of what must not pass, or put the rule in the chain carrying the policy it has to overrule, because an `accept` terminates only its own base chain and every other chain at that hook still runs, docker's `drop` policy on forwarded packets included. Reserved for an accept whose only job is to skip the fragment's own later rules. | diff --git a/roles/web-app-roulette-wheel/files/Dockerfile b/roles/web-app-roulette-wheel/files/Dockerfile index a2df2dfca1..349dae69b6 100644 --- a/roles/web-app-roulette-wheel/files/Dockerfile +++ b/roles/web-app-roulette-wheel/files/Dockerfile @@ -1,6 +1,12 @@ ARG ROULETTE_IMAGE ARG ROULETTE_VERSION FROM ${ROULETTE_IMAGE}:${ROULETTE_VERSION} + +RUN rm -rf /var/lib/apt/lists/* \ + && apt-get -o Acquire::Retries=3 update && apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ + curl \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /app ARG ROULETTE_WHEEL_APP_REL COPY ./${ROULETTE_WHEEL_APP_REL}/package.json ./${ROULETTE_WHEEL_APP_REL}/package-lock.json ./ diff --git a/roles/web-svc-simpleicons/files/Dockerfile b/roles/web-svc-simpleicons/files/Dockerfile index 4fc72dd4e9..9bbaaf8858 100644 --- a/roles/web-svc-simpleicons/files/Dockerfile +++ b/roles/web-svc-simpleicons/files/Dockerfile @@ -10,6 +10,11 @@ RUN npm install ARG container_port FROM ${SIMPLEICONS_IMAGE}:${SIMPLEICONS_VERSION} +RUN rm -rf /var/lib/apt/lists/* \ + && apt-get -o Acquire::Retries=3 update && apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ + curl \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /app COPY --from=builder /app/node_modules ./node_modules COPY ./config/server.js . diff --git a/tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py b/tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py new file mode 100644 index 0000000000..b0b701d5c4 --- /dev/null +++ b/tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py @@ -0,0 +1,164 @@ +"""A healthcheck that shells out to a binary must find it in the image it runs in. + +A ``-slim`` tag is the upstream's stripped variant: it carries the +runtime and nothing around it. ``node:26-bookworm`` ships ``/usr/bin/curl``, +``node:26-slim`` does not, and a ``healthcheck: flavor: curl`` renders +``CMD curl -f `` either way. The probe then fails from the first beat, and +the container never reports healthy. + +The two deploy modes answer differently, and neither answer names the binary. +``sys-svc-compose`` waits on the database container alone in compose and on +every stack task in swarm, so a permanently unhealthy service passes unnoticed +under compose and stops the deploy under swarm: the task dies with ``non-zero +exit (137): dockerexec: unhealthy container``, is rescheduled, dies again, and +the wait fails. Every deployment holding the service in its closure fails with +it, reported against the service that was waited for. + +Scope +===== +Roles whose ``meta/services.yml`` declares a healthcheck flavor on an image +whose ``version`` names a stripped variant. The binary is read from the probe +the declaration renders (:func:`utils.docker.healthcheck.compose.compose`), not +restated here, so a flavor that changes its command moves this check with it. A +flavor whose command is a shell builtin or the image's own interpreter needs no +package and is not checked. + +The role's ``files/Dockerfile`` must then install that binary. An image that +already ships it is not a defence: the tag is a moving target, and the role that +depends on the binary is the one that should say so. + +Per-role opt-out +================ +``# nocheck: healthcheck-binary`` on the ``flavor`` line or the one above it, +with a reason naming where the binary comes from instead. +""" + +from __future__ import annotations + +import re +import unittest + +from utils.annotations.suppress import is_suppressed_at +from utils.cache.files import read_text +from utils.cache.yaml import load_yaml_any +from utils.docker.healthcheck.compose import compose +from utils.roles.mapping import ROLE_FILE_META_SERVICES + +from . import PROJECT_ROOT + +RULE = "healthcheck-binary" + +STRIPPED = ("slim",) +"""Tag fragments that name an upstream's stripped variant. + +``alpine`` is deliberately out: measured against ``nginx:1.31.6-alpine``, the +image carries ``/usr/bin/curl`` as a real 272 KB binary and ``wget`` as a +busybox applet, so the four alpine roles that declare those flavors are served +by their base today. Widening this tuple to cover them needs that measurement +repeated per image, not an assumption from the word "alpine". +""" + +SHELL_BUILTINS = frozenset({"sh", "bash", "test", "true", "exit", "CMD-SHELL"}) +"""Probe commands that need no package: the image's own shell runs them.""" + + +def _probe_binary(flavor: object) -> str | None: + """The executable a declared flavor asks docker to run, if it is a program. + + Args: + flavor: the ``healthcheck.flavor`` value, one name or a list. + """ + try: + probe = compose(flavor, port=80, path="/", hostname="localhost") + test = probe.test() + except Exception: + return None + argv = [str(part) for part in test] + if not argv or argv[0] != "CMD" or len(argv) < 2: + return None + binary = argv[1] + return None if binary in SHELL_BUILTINS else binary + + +def _is_stripped(version: object) -> bool: + text = str(version) + return any(fragment in text for fragment in STRIPPED) + + +def _installs(dockerfile_body: str, binary: str) -> bool: + """Whether the Dockerfile names the binary as something it installs.""" + return ( + re.search(rf"(? dict[str, int]: + """The 1-based ``flavor:`` line of each top-level service key. + + ``meta/services.yml`` maps one service per top-level key, so the flavor a + key owns is the first one under it. Taking the file's first flavor line + instead would let one service's marker silence another's. + + Args: + lines: the file's lines, as :func:`read_text` splits them. + """ + found: dict[str, int] = {} + current: str | None = None + for number, line in enumerate(lines, start=1): + key = re.match(r"([A-Za-z0-9_.-]+):\s*(?:#.*)?$", line) + if key: + current = key.group(1) + continue + if current and current not in found and re.match(r"\s+flavor:", line): + found[current] = number + return found + + +class TestHealthcheckBinaryOnSlimBase(unittest.TestCase): + def test_a_stripped_base_installs_the_binary_its_healthcheck_runs(self): + offenders = [] + for services in sorted( + (PROJECT_ROOT / "roles").glob(f"*/{ROLE_FILE_META_SERVICES}") + ): + role_dir = services.parent.parent + data = load_yaml_any(str(services), default_if_missing={}) or {} + lines = read_text(str(services)).splitlines() + flavor_lines = _flavor_lines(lines) + for key, entry in data.items(): + if not isinstance(entry, dict): + continue + health = entry.get("healthcheck") + if not isinstance(health, dict) or "flavor" not in health: + continue + if not _is_stripped(entry.get("version", "")): + continue + binary = _probe_binary(health["flavor"]) + if binary is None: + continue + if is_suppressed_at(lines, flavor_lines.get(key, 0), RULE): + continue + dockerfile = role_dir / "files" / "Dockerfile" + body = read_text(str(dockerfile)) if dockerfile.is_file() else "" + if _installs(body, binary): + continue + offenders.append( + f"{role_dir.name}: service '{key}' declares healthcheck " + f"flavor {health['flavor']!r}, which runs {binary!r}, on the " + f"stripped image {entry.get('image')}:{entry.get('version')}; " + f"{'files/Dockerfile does not install it' if body else 'the role has no files/Dockerfile'}" + ) + + self.assertEqual( + [], + offenders, + "A stripped base carries the runtime and nothing around it, so a " + "healthcheck binary has to be installed by the role that depends on " + "it. Add it to the role's files/Dockerfile, pick a flavor the image " + f"can already run, or mark the line `# nocheck: {RULE}` with a " + "reason naming where the binary comes from:\n " + "\n ".join(offenders), + ) + + +if __name__ == "__main__": + unittest.main() From cc6da2417488cb2177483c0f79898091d2198338 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 03:28:12 +0200 Subject: [PATCH 122/197] test(lint): judge a healthcheck against what its base was measured to hold c31c62efa checked four of the repository's 86 healthcheck declarations. It read the probe's CMD form only, scoped itself to slim, and counted bash as a shell builtin, which is where most of the gap sat: tcp, connect and http_status all render CMD bash -c over /dev/tcp, so 31 declarations were skipped as needing nothing. It now examines 18, and what it still skips it skips for a reason. bash is a program like any other here. Its absence is not hypothetical: alpine has no /bin/bash at all, and its busybox /bin/sh does not implement /dev/tcp, so the redirection those three flavors are built on needs the real shell rather than any shell. No role pairs them today, which is why this is prevention and not a fix. The CMD-SHELL form is parsed instead of ignored: || reads as alternatives, && and ; as a sequence needing all its parts, and shell builtins are dropped so a trailing || exit 1 cannot make every requirement look satisfiable. That brings in nc, http and the msmtp compositions, six declarations the old shape could not see. http is the one that needed the alternatives: it runs wget or curl and is satisfied by either. alpine moves from a docstring exemption into the scope with a measured table behind it. Read out of the images with docker create and docker cp, which reports an absent path rather than a shell's opinion of it: debian slim carries /bin/bash at 1298416 bytes and none of curl, wget or nc; nginx alpine carries curl at 272432 bytes as a real binary, wget and nc as busybox applets, and no bash. Adding a family means measuring it the same way. Vendor images stay out. Their contents are not knowable from the tree and most carry no Dockerfile of ours, so the rule's remedy has nowhere to land; skipping them honestly beats a table that rots. Sixteen cases were run against the predicate, one per defect class and its satisfied counterpart. Two corrected the work: nc is a busybox applet on alpine, which the first table omitted and which made nc-on-alpine flag wrongly, and this file's own prose named a Debian codename, which test_no_debian_codename_tags caught in the gate. Co-Authored-By: Claude Opus 5 (1M context) --- .../test_healthcheck_binary_on_slim_base.py | 176 +++++++++++------- 1 file changed, 112 insertions(+), 64 deletions(-) diff --git a/tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py b/tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py index b0b701d5c4..e7b81a3df9 100644 --- a/tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py +++ b/tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py @@ -1,12 +1,12 @@ -"""A healthcheck that shells out to a binary must find it in the image it runs in. +"""A healthcheck must find every program it runs in the image it runs in. -A ``-slim`` tag is the upstream's stripped variant: it carries the -runtime and nothing around it. ``node:26-bookworm`` ships ``/usr/bin/curl``, -``node:26-slim`` does not, and a ``healthcheck: flavor: curl`` renders -``CMD curl -f `` either way. The probe then fails from the first beat, and -the container never reports healthy. +A stripped base carries the runtime and little else, and which "little else" +differs per family. The declaration does not know that: ``flavor: curl`` renders +``CMD curl -f `` and ``flavor: tcp`` renders ``CMD bash -c`` over +``/dev/tcp`` whatever the image underneath is. Where the program is absent the +probe fails from its first beat and the container never reports healthy. -The two deploy modes answer differently, and neither answer names the binary. +The two deploy modes answer differently, and neither answer names the program. ``sys-svc-compose`` waits on the database container alone in compose and on every stack task in swarm, so a permanently unhealthy service passes unnoticed under compose and stops the deploy under swarm: the task dies with ``non-zero @@ -14,23 +14,26 @@ exit (137): dockerexec: unhealthy container``, is rescheduled, dies again, and the wait fails. Every deployment holding the service in its closure fails with it, reported against the service that was waited for. -Scope -===== -Roles whose ``meta/services.yml`` declares a healthcheck flavor on an image -whose ``version`` names a stripped variant. The binary is read from the probe -the declaration renders (:func:`utils.docker.healthcheck.compose.compose`), not -restated here, so a flavor that changes its command moves this check with it. A -flavor whose command is a shell builtin or the image's own interpreter needs no -package and is not checked. +What is checked +=============== +Every ``meta/services.yml`` healthcheck whose service sits on a base this file +has measured. The programs come from the probe the declaration renders +(:func:`utils.docker.healthcheck.compose.compose`), both its ``CMD`` and its +``CMD-SHELL`` form, so a flavor that changes its command moves this check with +it rather than drifting from a table restated here. ``||`` in a shell probe is +read as alternatives, ``&&`` and ``;`` as a sequence that needs all of its +parts, and shell builtins are dropped so a trailing ``|| exit 1`` cannot make +every requirement look satisfiable. -The role's ``files/Dockerfile`` must then install that binary. An image that -already ships it is not a defence: the tag is a moving target, and the role that -depends on the binary is the one that should say so. +A requirement is met when the measured base provides the program or the role's +``files/Dockerfile`` installs it. A base that is neither family is skipped: a +vendor image's contents are not knowable from the tree, and for most of them +there is no Dockerfile of ours to add anything to. -Per-role opt-out -================ +Per-service opt-out +=================== ``# nocheck: healthcheck-binary`` on the ``flavor`` line or the one above it, -with a reason naming where the binary comes from instead. +with a reason naming where the program comes from instead. """ from __future__ import annotations @@ -48,49 +51,77 @@ from . import PROJECT_ROOT RULE = "healthcheck-binary" -STRIPPED = ("slim",) -"""Tag fragments that name an upstream's stripped variant. +PROVIDED: dict[str, frozenset[str]] = { + "slim": frozenset({"bash", "sh"}), + "alpine": frozenset({"sh", "curl", "wget", "nc"}), +} +"""What each stripped family ships, read out of the images themselves. -``alpine`` is deliberately out: measured against ``nginx:1.31.6-alpine``, the -image carries ``/usr/bin/curl`` as a real 272 KB binary and ``wget`` as a -busybox applet, so the four alpine roles that declare those flavors are served -by their base today. Widening this tuple to cover them needs that measurement -repeated per image, not an assumption from the word "alpine". +Measured with ``docker create`` plus ``docker cp``, which reports a path that is +absent instead of reporting a shell's opinion of it: + +* ``node:26-slim`` and ``python:3.13.15-slim`` carry ``/bin/bash`` at 1298416 + bytes and none of ``curl``, ``wget`` or ``nc``. The full ``node:26`` variant + does carry ``/usr/bin/curl`` at 280800 bytes, which is why moving a role onto + the stripped tag takes the program away without touching its healthcheck. +* ``nginx:1.31.6-alpine`` carries ``/usr/bin/curl`` at 272432 bytes as a real + binary, ``/usr/bin/wget`` and ``/usr/bin/nc`` as busybox applets, and has no + ``/bin/bash`` at all. Its ``/bin/sh`` is busybox, which does not implement + ``/dev/tcp``, so the flavors built on that redirection need the real shell + rather than any shell. + +Adding a family means measuring it the same way, not inferring it from the tag. """ -SHELL_BUILTINS = frozenset({"sh", "bash", "test", "true", "exit", "CMD-SHELL"}) -"""Probe commands that need no package: the image's own shell runs them.""" +SHELL_BUILTINS = frozenset({"exit", "true", "false", "test", "[", ":", "echo"}) +"""Words a probe may invoke that no package provides.""" -def _probe_binary(flavor: object) -> str | None: - """The executable a declared flavor asks docker to run, if it is a program. +def _families(version: object) -> list[str]: + """The measured families a version tag names, in declaration order.""" + text = str(version) + return [family for family in PROVIDED if family in text] + + +def _requirements(flavor: object) -> list[list[str]]: + """The programs a declared flavor runs, as alternatives of sequences. + + ``[["wget"], ["curl"]]`` means either one satisfies the probe; + ``[["msmtp", "curl"]]`` would mean it needs both. Args: flavor: the ``healthcheck.flavor`` value, one name or a list. """ try: - probe = compose(flavor, port=80, path="/", hostname="localhost") - test = probe.test() + test = [ + str(part) + for part in compose(flavor, port=80, path="/", hostname="h").test() + ] except Exception: - return None - argv = [str(part) for part in test] - if not argv or argv[0] != "CMD" or len(argv) < 2: - return None - binary = argv[1] - return None if binary in SHELL_BUILTINS else binary + return [] + if not test: + return [] + if test[0] == "CMD": + return [[test[1]]] if len(test) > 1 else [] + if len(test) < 2: + return [] + groups: list[list[str]] = [] + for alternative in re.split(r"\|\|", test[1]): + words = [ + segment.strip().split()[0] + for segment in re.split(r"&&|;", alternative) + if segment.strip() + ] + needed = [word for word in words if word not in SHELL_BUILTINS] + if needed: + groups.append(needed) + return groups -def _is_stripped(version: object) -> bool: - text = str(version) - return any(fragment in text for fragment in STRIPPED) - - -def _installs(dockerfile_body: str, binary: str) -> bool: - """Whether the Dockerfile names the binary as something it installs.""" - return ( - re.search(rf"(? bool: + """Whether the Dockerfile names the program as something it installs.""" + pattern = rf"(? dict[str, int]: @@ -116,7 +147,7 @@ def _flavor_lines(lines: list[str]) -> dict[str, int]: class TestHealthcheckBinaryOnSlimBase(unittest.TestCase): - def test_a_stripped_base_installs_the_binary_its_healthcheck_runs(self): + def test_a_stripped_base_provides_every_program_its_healthcheck_runs(self): offenders = [] for services in sorted( (PROJECT_ROOT / "roles").glob(f"*/{ROLE_FILE_META_SERVICES}") @@ -131,32 +162,49 @@ class TestHealthcheckBinaryOnSlimBase(unittest.TestCase): health = entry.get("healthcheck") if not isinstance(health, dict) or "flavor" not in health: continue - if not _is_stripped(entry.get("version", "")): + families = _families(entry.get("version", "")) + if not families: continue - binary = _probe_binary(health["flavor"]) - if binary is None: + groups = _requirements(health["flavor"]) + if not groups: continue if is_suppressed_at(lines, flavor_lines.get(key, 0), RULE): continue dockerfile = role_dir / "files" / "Dockerfile" body = read_text(str(dockerfile)) if dockerfile.is_file() else "" - if _installs(body, binary): + available = set().union(*(PROVIDED[family] for family in families)) + satisfied = any( + all( + program in available or _installs(body, program) + for program in group + ) + for group in groups + ) + if satisfied: continue + wanted = " or ".join(" and ".join(group) for group in groups) offenders.append( f"{role_dir.name}: service '{key}' declares healthcheck " - f"flavor {health['flavor']!r}, which runs {binary!r}, on the " - f"stripped image {entry.get('image')}:{entry.get('version')}; " - f"{'files/Dockerfile does not install it' if body else 'the role has no files/Dockerfile'}" + f"flavor {health['flavor']!r}, which runs {wanted}, on the " + f"{'/'.join(families)} image {entry.get('image')}:" + f"{entry.get('version')} that provides " + f"{', '.join(sorted(available))}; " + + ( + "files/Dockerfile installs none of them" + if body + else "the role has no files/Dockerfile" + ) ) self.assertEqual( [], offenders, - "A stripped base carries the runtime and nothing around it, so a " - "healthcheck binary has to be installed by the role that depends on " - "it. Add it to the role's files/Dockerfile, pick a flavor the image " - f"can already run, or mark the line `# nocheck: {RULE}` with a " - "reason naming where the binary comes from:\n " + "\n ".join(offenders), + "A stripped base carries the runtime and little else, so a program a " + "healthcheck runs has to come from the base or from the role that " + "depends on it. Install it in the role's files/Dockerfile, pick a " + "flavor the image can already run, or mark the line " + f"`# nocheck: {RULE}` with a reason naming where the program comes " + "from:\n " + "\n ".join(offenders), ) From 4b0c7423cbff0d90ef8fc26e5110b718747835eb Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 03:34:28 +0200 Subject: [PATCH 123/197] fix(lint): credit curl to nginx rather than to Alpine cc6da24174 measured Alpine on nginx:1.31.6-alpine alone and recorded curl as something the family ships. nginx installs it: /usr/bin/curl is there at 272432 bytes, and at 321880 bytes in the Debian nginx:1.31, while redis:8-alpine and node:26-alpine measured the same way have no curl at all. The table therefore passed a curl flavor on any alpine image, which is the failure the rule exists to catch, spelled with the rule's own name on it. The family now holds what busybox provides, sh, wget and nc. What one image adds on top of its family goes in IMAGE_PROVIDES, keyed on the image name with the registry prefix dropped, and nginx is its only entry today. The Debian half was re-measured while checking this, over /usr/bin, /bin, /usr/local/bin and /sbin rather than one path: debian:13-slim, node:26-slim and python:3.13.15-slim carry /bin/bash at 1298416 bytes and none of curl, wget or nc. bash is Essential in Debian and the other three are not, which is the whole difference between the stripped tag and the full one. Nine cases ran against the predicate. The two that matter are curl on redis:8-alpine and on node:26-alpine, which passed before this commit and flag after it; curl on nginx still passes, and the six alpine declarations the tree actually holds stay green, fediwall and nextcloud on nginx's curl, hugo and littlejs on busybox wget, taiga on either. Co-Authored-By: Claude Opus 5 (1M context) --- .../test_healthcheck_binary_on_slim_base.py | 51 ++++++++++++++----- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py b/tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py index e7b81a3df9..a883650bcc 100644 --- a/tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py +++ b/tests/lint/ansible/roles/meta/test_healthcheck_binary_on_slim_base.py @@ -53,24 +53,40 @@ RULE = "healthcheck-binary" PROVIDED: dict[str, frozenset[str]] = { "slim": frozenset({"bash", "sh"}), - "alpine": frozenset({"sh", "curl", "wget", "nc"}), + "alpine": frozenset({"sh", "wget", "nc"}), } """What each stripped family ships, read out of the images themselves. Measured with ``docker create`` plus ``docker cp``, which reports a path that is -absent instead of reporting a shell's opinion of it: +absent instead of reporting a shell's opinion of it. Every lookup covered +``/usr/bin``, ``/bin``, ``/usr/local/bin`` and ``/sbin``: -* ``node:26-slim`` and ``python:3.13.15-slim`` carry ``/bin/bash`` at 1298416 - bytes and none of ``curl``, ``wget`` or ``nc``. The full ``node:26`` variant - does carry ``/usr/bin/curl`` at 280800 bytes, which is why moving a role onto - the stripped tag takes the program away without touching its healthcheck. -* ``nginx:1.31.6-alpine`` carries ``/usr/bin/curl`` at 272432 bytes as a real - binary, ``/usr/bin/wget`` and ``/usr/bin/nc`` as busybox applets, and has no - ``/bin/bash`` at all. Its ``/bin/sh`` is busybox, which does not implement - ``/dev/tcp``, so the flavors built on that redirection need the real shell - rather than any shell. +* ``debian:13-slim``, ``node:26-slim`` and ``python:3.13.15-slim`` carry + ``/bin/bash`` at 1298416 bytes and none of ``curl``, ``wget`` or ``nc``. bash + is Essential in Debian and the other three are not, which is the whole + difference between the stripped tag and the full one: the full ``node:26`` + variant carries ``/usr/bin/curl`` at 280800 bytes. +* ``redis:8-alpine`` and ``node:26-alpine`` carry ``/usr/bin/wget`` and + ``/usr/bin/nc`` as busybox applets, no ``curl``, and no ``/bin/bash`` at all. + Their ``/bin/sh`` is busybox, which does not implement ``/dev/tcp``, so the + flavors built on that redirection need the real shell rather than any shell. -Adding a family means measuring it the same way, not inferring it from the tag. +The family is read from the tag, which holds for every image in the tree today: +every ``slim`` tag here is Debian based and every ``alpine`` tag is Alpine +based. Adding a family means measuring it the same way, not inferring it. +""" + +IMAGE_PROVIDES: dict[str, frozenset[str]] = { + "nginx": frozenset({"curl"}), +} +"""Programs an individual image adds on top of what its family ships. + +nginx installs curl in both of its variants: ``/usr/bin/curl`` at 272432 bytes +in ``nginx:1.31.6-alpine`` and at 321880 bytes in the Debian ``nginx:1.31``. +Base Alpine does not have it at all, measured the same way on ``redis:8-alpine`` +and ``node:26-alpine``. Reading nginx's curl as a property of Alpine would pass +a ``curl`` flavor on any alpine image, so it is recorded against the image that +installs it. """ SHELL_BUILTINS = frozenset({"exit", "true", "false", "test", "[", ":", "echo"}) @@ -83,6 +99,16 @@ def _families(version: object) -> list[str]: return [family for family in PROVIDED if family in text] +def _image_extras(image: object) -> frozenset[str]: + """What this particular image adds beyond its family. + + Args: + image: the ``image`` value, with or without a registry prefix. + """ + name = str(image).rsplit("/", 1)[-1] + return IMAGE_PROVIDES.get(name, frozenset()) + + def _requirements(flavor: object) -> list[list[str]]: """The programs a declared flavor runs, as alternatives of sequences. @@ -173,6 +199,7 @@ class TestHealthcheckBinaryOnSlimBase(unittest.TestCase): dockerfile = role_dir / "files" / "Dockerfile" body = read_text(str(dockerfile)) if dockerfile.is_file() else "" available = set().union(*(PROVIDED[family] for family in families)) + available |= _image_extras(entry.get("image", "")) satisfied = any( all( program in available or _installs(body, program) From df19dd103ca1afb027d4a776143ac9f9329b8065 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 03:49:39 +0200 Subject: [PATCH 124/197] fix(matrix): match the invite tile on the button, not on a row that is absent 9661800cb0 scoped the tile to [role="row"][aria-level="2"], taken from upstream's test helpers for Element 1.12.28. meta/services.yml pinned v1.12.27 then and has never held 1.12.28 on any ref, so the selector described markup the deployment does not render. Read out of biber's DOM in the trace of the failing job: role="row" occurs zero times, aria-expanded zero times, aria-label="Toggle " zero times, and every aria-level in the document is "1" on a heading. The only aria-level="2" in the whole trace is inside the test's own injected script. The invite is a bare button in a virtuoso list: DIV[data-testid="virtuoso-item-list"] > DIV[data-index="0"] > BUTTON[aria-label="Open room Administrator Infinito.Nexus invitation."] So the tile was never clicked, biber stayed on the welcome screen, and the poll spent its 120 seconds on a page that could not change. The screenshots of all three attempts show the same state: the invite row visible in the list, a device-verification prompt beside it, no room open. Matching the button directly drops the dependency on the wrapper. The label prefix and the partner match are unchanged, and "Open room settings" is the only other button carrying the prefix, which the partner match excludes. The section expansion stays. It is a no-op against 1.12.27, where nothing reports aria-expanded, and it is what upstream's helpers describe for the release after it. Not verified: the accept step behind the tile. biber never reached the invite view in any attempt, so which of "Start chatting", "Accept" or "Join" 1.12.27 renders there is unobserved, and the anchored regex covering all three is carried over untested. Co-Authored-By: Claude Opus 5 (1M context) --- .../web-app-matrix/files/playwright/test-dm-admin-biber.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/web-app-matrix/files/playwright/test-dm-admin-biber.js b/roles/web-app-matrix/files/playwright/test-dm-admin-biber.js index 0ba3371a86..d60069523d 100644 --- a/roles/web-app-matrix/files/playwright/test-dm-admin-biber.js +++ b/roles/web-app-matrix/files/playwright/test-dm-admin-biber.js @@ -100,8 +100,6 @@ exports.register = function (shared) { await expect .poll(async () => { return await biberPage.evaluate(() => { - // Pending invites sit in a section that renders collapsed, so the - // invite row is absent from the DOM until it is expanded. const sections = document.querySelectorAll( 'button[aria-expanded="false"][aria-label^="Toggle "]', ); @@ -109,8 +107,10 @@ exports.register = function (shared) { section.click(); } + // Not scoped to a row wrapper: the list is virtualised and 1.12.27 + // renders no role=row, aria-level or aria-expanded at all. const roomTiles = document.querySelectorAll( - '[role="row"][aria-level="2"] button[aria-label^="Open room"]', + 'button[aria-label^="Open room"]', ); for (const tile of roomTiles) { const name = `${tile.getAttribute("aria-label") || ""} ${tile.textContent || ""}`; From 94429e8fc52eacbc8a5a71f0a0962b525618032a Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 03:52:26 +0200 Subject: [PATCH 125/197] feat(matrix): take the Element release that has the invites section v1.12.28 is the newest tag vectorim/element-web publishes. Read at the artifact rather than at the tag: /app/version inside the image reads 1.12.28, and the control on v1.12.27 reads 1.12.27, so the tag does not lie here. /usr/bin/wget is a busybox applet in it, which the wget healthcheck flavor needs. It carries 'Room list: add an Invites section' (#34736), the section 9661800cb0 wrote its selector for while the pin stood at v1.12.27 and rendered no such thing. df19dd103 made that selector independent of the wrapper, so the expansion loop beside it stops being a no-op here instead of the tile becoming unreachable again. It also carries a fix for GHSA-wqmv-r2qj-2j9p. What the bump can reach is bounded and was counted rather than assumed. Seven of the twelve matrix specs hold no DOM selector at all and speak only to synapse, which stays at v1.160.0. The exposure is _shared.js with 32, this role's DM spec with 9 and the seaweedfs spec with 2. Of the shared helper's selectors, ten carry a fallback chain and three are single-point, none of those three on an internal mx_ class. meta/csp.yml grants element unsafe-inline, unsafe-eval and connect-src *, with no hash pin, so a changed asset bundle cannot break the policy. Not verified: no deploy ran. The same release reworks the room list three ways -- the invites section, collapsing sections on drag, a default people section -- and that is the area the DM spec drives, so the specs that touch it are the ones to read first if the next round is red. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-matrix/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-matrix/meta/services.yml b/roles/web-app-matrix/meta/services.yml index 37d6a76c7a..232a0b2a34 100644 --- a/roles/web-app-matrix/meta/services.yml +++ b/roles/web-app-matrix/meta/services.yml @@ -66,7 +66,7 @@ synapse: element: healthcheck: flavor: wget - version: v1.12.27 + version: v1.12.28 image: vectorim/element-web name: matrix-element ports: From bde617af95af9bcc9333f242dab98c95bf3ad548 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 09:23:07 +0200 Subject: [PATCH 126/197] fix(bigbluebutton): take dockerize from the registry that publishes both arches da665ab699 pinned the helper to jwilder/dockerize:v0.11.0 on Docker Hub, which is the one tag in that repository's history published for arm64 alone. The runners are amd64, so generate-compose died on 'exec /bin/dockerize: exec format error' at bootstrap_compose_project.yml:13 and took the deploy with it. Not only BigBlueButton's own: Nextcloud#11 failed on the same line, because BBB sits in its closure. The mirror is not at fault. Both the mirrored copy and the upstream tag are single manifests, so the copy is faithful; Docker Hub simply has no amd64 for that tag. Measured twice by different means, the registry API reading the config blob and a pull reading the daemon's view, agreeing on all eighteen rows: v0.11.0 is linux/arm64, v0.10.0 is linux/amd64, and the fifteen alangecker/bbb-docker-* images are linux/amd64 without exception. GHCR publishes what Docker Hub does not: ghcr.io/jwilder/dockerize carries a manifest list with linux/amd64 and linux/arm64 for v0.11.0 and for v0.15.1, and its tags run four minor versions further than Docker Hub's, which upstream's own issue #320 describes as six years stale. ghcr.io is already in _MIRRORABLE_REGISTRIES and eight other roles pin through it, so nothing in the mirror chain changes. The jump from v0.11.0 to v0.15.1 is covered by what the script actually uses: generate-compose invokes dockerize with -template and nothing else, and v0.15.1 documents that flag unchanged, single-argument form included. Verified by deploying it. web-app-bigbluebutton variant 1 in compose, the combination CI failed on, with make test green beforehand: the pre-pull takes ghcr.io/jwilder/dockerize:v0.15.1, the task that used to die finishes in 0.60s, 'exec format error' appears zero times in 104k lines of log, and the play ends ok=1178 changed=88 failed=0. This does not make BigBlueButton run on arm64 and does not claim to. Fifteen of its sixteen images are amd64-only; dockerize was the single arm64 one in the set, which is why it broke an otherwise uniformly amd64 stack. Variant 0 and the swarm mode are unexercised. Co-Authored-By: Claude Opus 5 (1M context) --- roles/web-app-bigbluebutton/meta/services.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/web-app-bigbluebutton/meta/services.yml b/roles/web-app-bigbluebutton/meta/services.yml index 601e464673..85af312fbb 100644 --- a/roles/web-app-bigbluebutton/meta/services.yml +++ b/roles/web-app-bigbluebutton/meta/services.yml @@ -194,8 +194,8 @@ bbb-webrtc-recorder: # nocheck: playwright-service-flag: script-time helper of upstream generate-compose, mirror pre-pull only # nocheck: dynamic-flag: fixed upstream-compose helper, not group-gated dockerize: - image: jwilder/dockerize - version: "v0.11.0" + image: ghcr.io/jwilder/dockerize + version: "v0.15.1" enabled: false # nocheck: playwright-service-flag, dynamic-flag infrastructure backup consumer From 55b5a3dd93684f76437d897c62436cff713f9139 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 10:52:02 +0200 Subject: [PATCH 127/197] fix(fider): run the build that reads the private-network opt-in 0d1ba1777f set ALLOW_PRIVATE_NETWORK_TARGETS=true in July with the right diagnosis: Fider's SSRF guard refuses the OAuth token exchange when the Keycloak host resolves to an internal address, so login 500s and never completes. The image pin stayed at v0.36.0, which does not read that variable, so the fix has been inert ever since. Read out of the binaries rather than the release notes: `stable` contains the string ALLOW_PRIVATE_NETWORK_TARGETS, v0.36.0 contains it zero times. Upstream added the opt-in on 2026-06-24 and the check now returns before the hostname is resolved, which is what puts the OAuth token path under it as well. CI showed this as three red specs on Fider#0, admin sso login, biber sso login and the seaweedfs tenant-logo upload, all with a missing .c-menu-user. The artifact says why the selector is missing: the page is Fider's own 500 screen, and fider.log carries the cause four times over, `Token URL is not allowed: This URL is not allowed because it targets a private or internal network address (app/services/oauth/oauth.go:320)`. The selector was right; there was no session to render. The tag is `stable` and not a version because upstream published v0.36.1 on GitHub on 2026-07-03 and never pushed the matching image. Docker Hub's newest release tag remains v0.36.0, and `stable` was rebuilt three days after that release, carrying the opt-in. The marker on the line records both that and the condition for removing it. This is a moving tag in a repository that has just finished pinning fifteen of them, and the mirror copies by tag, so an existing copy of `stable` can go quietly stale. That is the accepted cost of the only build upstream ships with the fix, and the TODO is what closes it. audit.md moves with it: the report is generated from the pins and the lint fails when the committed copy differs from a fresh generation. Co-Authored-By: Claude Opus 5 (1M context) --- docs/contributing/design/role/services/mcp/audit.md | 2 +- roles/web-app-fider/meta/services.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing/design/role/services/mcp/audit.md b/docs/contributing/design/role/services/mcp/audit.md index 31dcaf1dbc..f9b15c0fa0 100644 --- a/docs/contributing/design/role/services/mcp/audit.md +++ b/docs/contributing/design/role/services/mcp/audit.md @@ -15,7 +15,7 @@ not the version the MCP surface was verified against. | `svc-db-qdrant` | v1.19.1 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-svc-db-qdrant`) | read-only | 2 | adapter | `web-app-flowise`, `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-baserow` | 2.3.3 | [link](https://baserow.io/user-docs/mcp-server) | streamable_http | bearer_token | service_account (`mcp-web-app-baserow`) | read-only | 4 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-checkmk` | 2.4.0p32 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-checkmk`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | -| `web-app-fider` | v0.36.0 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-fider`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | +| `web-app-fider` | stable | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-fider`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-flowise` | 24-slim | [link](https://github.com/FlowiseAI/Flowise/tree/flowise%403.1.4/packages/server/src/routes/custom-mcp-servers) | โ€” | โ€” | โ€” | read-only | โ€” | native | โ€” | โ€” | | `web-app-gitea` | 1.27.3 | [link](https://pkg.go.dev/gitea.com/gitea/gitea-mcp) | streamable_http | bearer_token | service_account (`mcp-web-app-gitea`) | read-only | 24 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-gitlab` | โ€” | [link](https://docs.gitlab.com/user/gitlab_duo/model_context_protocol/mcp_server/) | streamable_http | bearer_token | service_account (`mcp-web-app-gitlab`) | read-only | 14 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | diff --git a/roles/web-app-fider/meta/services.yml b/roles/web-app-fider/meta/services.yml index eab202dadd..072028c8e7 100644 --- a/roles/web-app-fider/meta/services.yml +++ b/roles/web-app-fider/meta/services.yml @@ -36,7 +36,7 @@ fider: swarm: enabled: true image: getfider/fider - version: v0.36.0 + version: stable # nocheck: docker-version upstream released v0.36.1 on GitHub but never pushed that tag; its build ships only as `stable`, and v0.36.0 predates ALLOW_PRIVATE_NETWORK_TARGETS, without which the OAuth token exchange against an internally resolving Keycloak is refused. TODO: pin the semver tag once upstream publishes one. name: fider min_storage: 400MB backup: From 2567c5ece1e63f9b6a0482a6a313fa5a47da6bf6 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 11:25:12 +0200 Subject: [PATCH 128/197] feat(ci): let a manual run set the chunk size the formula would derive chunk_size() multiplied the runner slots a chunk may assume by the waves the queue window affords, and nothing could say otherwise. The derivation is the right default -- too few rows leaves runners idle, too many push a chunk's tail past the queue cancel -- but a run that wants smaller chunks had to edit default.env to get them. INFINITO_CI_CHUNK_SIZE now takes precedence when it is greater than zero, and zero keeps the derivation. The input travels the same path offset already travels: entry-manual-steer.yml offers the field and maps empty to 0, call-orchestrator.yml hands it to all four chunk blocks, call-test-deploy.yml exports it as the env var. It is a string rather than a number for the reason resume_from_chunk already documents: a workflow_dispatch number input reaches the caller as a string, and a number-typed workflow_call input then makes GitHub refuse the job before it exists. Two places in call-test-deploy.yml set it, not one. The plan summary resolves the chunk size itself, so exporting it only for the app resolution would print a plan that disagrees with the matrix the run then deploys. Measured against the live helper: 0 gives 80 rows over 3 chunks, 25 gives 25 over 4, and returning to 0 restores 80 and 3. The gate found three faults in the first attempt and all three are in this commit: the key sorted after INFINITO_CI_CONCURRENCY rather than before it, the dispatch description ran to 81 characters against a limit of 80, and the retrigger fixture did not carry the new input. That last one matters beyond the test: it is the check that would otherwise let a retrigger drop the chunk size silently. carried_inputs() itself needed nothing, since it derives from the declared inputs. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/call-orchestrator.yml | 9 +++++++++ .github/workflows/call-test-deploy.yml | 7 +++++++ .github/workflows/entry-manual-steer.yml | 6 ++++++ cli/meta/ci/slots.py | 10 +++++++++- default.env | 3 +++ .../meta/workflows/test_ci_retrigger_inputs.py | 1 + utils/env/handlers/passthrough.py | 1 + 7 files changed, 36 insertions(+), 1 deletion(-) diff --git a/.github/workflows/call-orchestrator.yml b/.github/workflows/call-orchestrator.yml index b71a3b4879..f37831656c 100644 --- a/.github/workflows/call-orchestrator.yml +++ b/.github/workflows/call-orchestrator.yml @@ -37,6 +37,11 @@ on: required: false type: string default: "0" + chunk_size: + description: "๐Ÿ“ฆ Rows one chunk holds. Empty or 0 derives it from the runner slots a chunk may assume and the waves the queue window affords. Typed string for the same reason as resume_from_chunk." + required: false + type: string + default: "0" resume_from_chunk: description: "โญ๏ธ Skip every chunk below this index. Use to continue a sweep after fixing what broke it, instead of re-running the green chunks. Typed string, not number: a workflow_dispatch number input reaches the caller as a string, and handing that to a number-typed workflow_call input makes GitHub reject the job before it is created." required: false @@ -174,6 +179,7 @@ jobs: index: "0" sweep: ${{ inputs.sweep != '' && inputs.sweep || github.run_number }} offset: ${{ inputs.offset }} + chunk_size: ${{ inputs.chunk_size }} whitelist: ${{ inputs.whitelist }} priority: ${{ inputs.priority }} lifecycles: ${{ inputs.lifecycles }} @@ -205,6 +211,7 @@ jobs: index: "1" sweep: ${{ inputs.sweep != '' && inputs.sweep || github.run_number }} offset: ${{ inputs.offset }} + chunk_size: ${{ inputs.chunk_size }} whitelist: ${{ inputs.whitelist }} priority: ${{ inputs.priority }} lifecycles: ${{ inputs.lifecycles }} @@ -238,6 +245,7 @@ jobs: index: "2" sweep: ${{ inputs.sweep != '' && inputs.sweep || github.run_number }} offset: ${{ inputs.offset }} + chunk_size: ${{ inputs.chunk_size }} whitelist: ${{ inputs.whitelist }} priority: ${{ inputs.priority }} lifecycles: ${{ inputs.lifecycles }} @@ -273,6 +281,7 @@ jobs: index: "3" sweep: ${{ inputs.sweep != '' && inputs.sweep || github.run_number }} offset: ${{ inputs.offset }} + chunk_size: ${{ inputs.chunk_size }} whitelist: ${{ inputs.whitelist }} priority: ${{ inputs.priority }} lifecycles: ${{ inputs.lifecycles }} diff --git a/.github/workflows/call-test-deploy.yml b/.github/workflows/call-test-deploy.yml index c3ce5cd652..8cd37c3bf6 100644 --- a/.github/workflows/call-test-deploy.yml +++ b/.github/workflows/call-test-deploy.yml @@ -21,6 +21,11 @@ on: required: false type: string default: "0" + chunk_size: + description: "Rows one chunk holds; 0 derives it from the runner slots and the queue window." + required: false + type: string + default: "0" whitelist: description: "Optional space-separated whitelist of app ids" required: false @@ -189,12 +194,14 @@ jobs: INFINITO_CI_SWEEP: ${{ inputs.sweep }} INFINITO_CI_CHUNK: ${{ inputs.index }} INFINITO_CI_OFFSET: ${{ inputs.offset }} + INFINITO_CI_CHUNK_SIZE: ${{ inputs.chunk_size }} run: ./scripts/github/resolve/output_apps.sh - name: ๐Ÿ—บ๏ธ Render the plan into the run summary shell: bash env: INFINITO_CI_SWEEP: ${{ inputs.sweep }} + INFINITO_CI_CHUNK_SIZE: ${{ inputs.chunk_size }} PLAN_DISTROS: ${{ inputs.distros }} PLAN_FILESYSTEM: ${{ inputs.filesystem }} PLAN_WHITELIST: ${{ steps.resolve_wl.outputs.whitelist }} diff --git a/.github/workflows/entry-manual-steer.yml b/.github/workflows/entry-manual-steer.yml index 8fd1c6f438..edcad68b70 100644 --- a/.github/workflows/entry-manual-steer.yml +++ b/.github/workflows/entry-manual-steer.yml @@ -58,6 +58,11 @@ on: required: false default: "0" type: string + chunk_size: + description: "๐Ÿ“ฆ Rows one chunk holds. Empty: derive it from slots and queue window" + required: false + default: "" + type: string chunk_gate: description: "๐Ÿ›‘ Stop the chunk chain at its first failed chunk" required: false @@ -107,6 +112,7 @@ jobs: priority: ${{ inputs.priority }} sweep: ${{ inputs.sweep }} offset: ${{ inputs.offset }} + chunk_size: ${{ inputs.chunk_size != '' && inputs.chunk_size || '0' }} chunk_gate: ${{ inputs.chunk_gate }} filesystem: ${{ inputs.filesystem }} workspace: ${{ inputs.workspace }} diff --git a/cli/meta/ci/slots.py b/cli/meta/ci/slots.py index 2a59db7fd0..bffd2df69e 100644 --- a/cli/meta/ci/slots.py +++ b/cli/meta/ci/slots.py @@ -142,7 +142,15 @@ def waves(repo_root: Path | None = None) -> int: def chunk_size(repo_root: Path | None = None) -> int: - """Rows one chunk may hold before its tail risks the queue cancel.""" + """Rows one chunk may hold before its tail risks the queue cancel. + + ``INFINITO_CI_CHUNK_SIZE`` takes precedence when set; 0 derives the value + from the runner slots a chunk may assume and the waves the queue window + affords. + """ + override = setting("INFINITO_CI_CHUNK_SIZE") + if override > 0: + return override return setting("INFINITO_CI_CONCURRENCY") * waves(repo_root) diff --git a/default.env b/default.env index e98136c83e..f079196ae4 100644 --- a/default.env +++ b/default.env @@ -106,6 +106,9 @@ INFINITO_CACHE_STACK=true # Index of the deploy chunk the discovery emits; each chunk block of call-orchestrator.yml passes its own. INFINITO_CI_CHUNK=0 +# Rows one deploy chunk holds, overriding the concurrency-times-waves derivation; 0 derives it. +INFINITO_CI_CHUNK_SIZE=0 + # Runner slots a deploy chunk may assume are free; with the job timeout it sizes a chunk in cli.meta.ci.slots. INFINITO_CI_CONCURRENCY=20 diff --git a/tests/integration/meta/workflows/test_ci_retrigger_inputs.py b/tests/integration/meta/workflows/test_ci_retrigger_inputs.py index 20c937aac2..f5588594c2 100644 --- a/tests/integration/meta/workflows/test_ci_retrigger_inputs.py +++ b/tests/integration/meta/workflows/test_ci_retrigger_inputs.py @@ -43,6 +43,7 @@ _VALUES = { "filesystem": "btrfs", "tor": "enforced", "offset": "40", + "chunk_size": "25", "chunk_gate": "false", "sweep": "7", "workspace": "true", diff --git a/utils/env/handlers/passthrough.py b/utils/env/handlers/passthrough.py index 8d563d5875..9509a7b7fb 100644 --- a/utils/env/handlers/passthrough.py +++ b/utils/env/handlers/passthrough.py @@ -95,6 +95,7 @@ STATIC_KEYS = ( "INFINITO_BLACKLIST", "INFINITO_PRIORITY", "INFINITO_CI_CHUNK", + "INFINITO_CI_CHUNK_SIZE", "INFINITO_CI_CONCURRENCY", "INFINITO_CI_DISTRO_BUDGET_SECONDS", "INFINITO_CI_JOB_LIMIT", From 57fbd3f37ad762c7e036425fbd49ef8a8456b2c5 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 16:54:21 +0200 Subject: [PATCH 129/197] fix(ci): dispatch the release workflow by the name it now has cron-release-highest.yml ran `gh workflow run release-version.yml`, a file renamed to call-release-version.yml by 71888d86af on 2026-08-07. GitHub does not error on a stale name: it resolves it against the deleted workflow's tombstone (upstream id 226809803, fork 248877824, both state=deleted) and answers HTTP 422. The step is the job's last, so the whole run went red. The last 25 scheduled upstream runs failed there, the streak reaching back to 2026-08-09. This removes the 422. It does not make releases green: workflow 226809803 last succeeded on 2026-07-08, and from 2026-07-09 through 2026-08-08 the dispatch worked while every run still failed at "Build missing CI images (ubuntu)". Expect that failure to surface next. test_workflow_references.py was meant to survive exactly this rename and could not: its pattern required a directory prefix, so a bare filename was invisible, and it reported zero offenders against twenty real ones. It now also matches a bare *.yml inside .github/workflows, where every such token names a sibling, and it checks both manual-dispatch forms, `gh workflow run` and a REST POST to actions/workflows//dispatches, for existence and for a workflow_dispatch trigger. Outside that directory the prefix stays mandatory, because a bare name collides with services.yml and compose.yml. The nineteen README labels are the same sweep's leftovers in the mermaid diagrams; the widened lint fails without them. Negative controls executed per branch: reverting the dispatch name, and pointing the REST call at a missing workflow and at a reusable-only one, each fails the matching check, and restoring makes them pass. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/README.md | 34 ++++---- .github/workflows/cron-release-highest.yml | 2 +- .../repository/test_workflow_references.py | 85 ++++++++++++++++--- 3 files changed, 92 insertions(+), 29 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 51196e8af2..bd2ad4ef11 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -13,21 +13,21 @@ flowchart TB pr["pull_request: opened, synchronize, reopened, ready_for_review"] --> eprc["entry-pr-change-orchestrate.yml"] dispatch["workflow_dispatch"] --> eman["entry-manual-steer.yml"] - epl --> orch["ci-orchestrator.yml"] - epl -->|"version tag on main"| relv["release-version.yml"] + epl --> orch["call-orchestrator.yml"] + epl -->|"version tag on main"| relv["call-release-version.yml"] eprc --> orch - eprc -->|"fork PRs: privileged prebuild"| imgbuild["images-build-ci.yml"] - eprc -->|"fork PRs: privileged prebuild"| imgmirror["images-mirror-missing.yml"] + eprc -->|"fork PRs: privileged prebuild"| imgbuild["call-images-build-ci.yml"] + eprc -->|"fork PRs: privileged prebuild"| imgmirror["call-images-mirror-missing.yml"] eman --> orch - subgraph orchestrator["ci-orchestrator.yml jobs"] + subgraph orchestrator["call-orchestrator.yml jobs"] waitfork["wait-fork-prereq-run"] --> forkready["fork-prereqs-ready"] - lintwf["lint.yml: make lint + hadolint"] - testwf["test.yml: make test"] + lintwf["call-lint.yml: make lint + hadolint"] + testwf["call-test.yml: make test"] codeql["cron-security-codeql.yml"] - buildci["build-ci-images: images-build-ci.yml"] --> dns["test-dns.yml"] - mirror["images-mirror-missing.yml"] + buildci["build-ci-images: call-images-build-ci.yml"] --> dns["call-test-dns.yml"] + mirror["call-images-mirror-missing.yml"] subgraph chain["serial chunk chain"] chunk0["test-deploy-chunk-0"] @@ -42,12 +42,12 @@ flowchart TB mirror --> chain buildci --> chain - chunk0 --> smoke["test-runner-smoke.yml"] + chunk0 --> smoke["call-test-runner-smoke.yml"] chain --> report["report-main-failures"] - instmake["test-install-make.yml"] - instpkgmgr["test-install-pkgmgr.yml"] - mirror --> devenv["test-workspace: test-workspace.yml"] + instmake["call-test-install-make.yml"] + instpkgmgr["call-test-install-pkgmgr.yml"] + mirror --> devenv["test-workspace: call-test-workspace.yml"] chain --> donegate["done"] smoke --> donegate @@ -301,12 +301,12 @@ flowchart TB pushmain["push: main"] --> updatewf prtarget["pull_request_target: opened, reopened"] --> depclose["entry-pr-open-dependabot-close.yml"] - relhighest -.->|"gh workflow run"| relver["release-version.yml"] - relver --> imgbuildci["images-build-ci.yml"] - manual["workflow_dispatch"] --> mirrorcleanup["images-mirror-cleanup.yml"] + relhighest -.->|"gh workflow run"| relver["call-release-version.yml"] + relver --> imgbuildci["call-images-build-ci.yml"] + manual["workflow_dispatch"] --> mirrorcleanup["entry-manual-mirror-cleanup.yml"] ``` Also manually dispatchable: `cron-images-mirror-all.yml`, `cron-images-cleanup-ci.yml`, `cron-cleanup-stale.yml`, `cron-update.yml`, `cron-release-highest.yml`, `call-release-version.yml`, `call-lint.yml`, `call-test.yml`, `call-test-dns.yml`, -`test-workspace.yml`, `test-runner-smoke.yml`. +`call-test-workspace.yml`, `call-test-runner-smoke.yml`. diff --git a/.github/workflows/cron-release-highest.yml b/.github/workflows/cron-release-highest.yml index 75183086d3..8e17fadbae 100644 --- a/.github/workflows/cron-release-highest.yml +++ b/.github/workflows/cron-release-highest.yml @@ -79,4 +79,4 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ steps.pick.outputs.tag }} - run: gh workflow run release-version.yml -f tag="${TAG}" + run: gh workflow run call-release-version.yml -f tag="${TAG}" diff --git a/tests/lint/repository/test_workflow_references.py b/tests/lint/repository/test_workflow_references.py index fcb8156f37..10340731fb 100644 --- a/tests/lint/repository/test_workflow_references.py +++ b/tests/lint/repository/test_workflow_references.py @@ -2,9 +2,17 @@ Scope ===== -Any occurrence of ``.github/workflows/.yml`` or ``actions/workflows/ -.yml`` in a tracked text file: `uses:` lines, Python constants, Makefile -variables, shell API URLs and Markdown links alike. +Three patterns: + +* ``.github/workflows/.yml`` or ``actions/workflows/.yml`` anywhere + in a tracked text file: `uses:` lines, Python constants, Makefile variables, + shell API URLs and Markdown links alike. +* a **bare** ``.yml`` inside ``.github/workflows/`` itself, where every + such token names a sibling workflow. Elsewhere in the tree a bare name is + ambiguous with role and compose files, so the prefix stays mandatory there. +* the target of a manual dispatch anywhere, which must exist **and** declare + ``workflow_dispatch``. Both forms count: ``gh workflow run .yml`` and a + REST POST to ``actions/workflows/.yml/dispatches``. Why === @@ -13,6 +21,11 @@ the run, a Python constant raises at test time, but a Markdown link or an API URL in a shell script fails only in production, months later. Matching the string against the directory listing catches all of them at once. +A stale `gh workflow run` target is the quietest of the three. GitHub resolves +the name against the deleted workflow's tombstone instead of erroring, answers +HTTP 422, and keeps the tombstone out of `GET /actions/workflows`, so listing a +repository's workflows shows a clean set while every dispatch fails. + A file opts out with a ``nocheck: workflow-references`` marker in its first 30 lines, for documents that name a workflow that does not exist yet. """ @@ -32,16 +45,30 @@ _NOCHECK_RULE = "workflow-references" _WORKFLOW_DIR = PROJECT_ROOT / ".github" / "workflows" _SUFFIXES = {".md", ".yml", ".yaml", ".py", ".sh", ".j2", ".rst", ".txt"} _REFERENCE_RE = re.compile(r"(?:\.github|actions)/workflows/([A-Za-z0-9._-]+\.ya?ml)") +_BARE_RE = re.compile(r"(?[A-Za-z0-9._-]+\.ya?ml)" + r"|actions/workflows/(?P[A-Za-z0-9._-]+\.ya?ml)/dispatches" +) def _scanned(path: Path) -> bool: return path.suffix in _SUFFIXES or path.name == "Makefile" -def missing_references() -> list[str]: - """Every `:: ` whose workflow is not on disk.""" - known = {entry.name for entry in _WORKFLOW_DIR.iterdir() if entry.is_file()} - offenders: list[str] = [] +def _known() -> set[str]: + return {entry.name for entry in _WORKFLOW_DIR.iterdir() if entry.is_file()} + + +def _patterns(path: Path) -> tuple[re.Pattern[str], ...]: + if path.is_relative_to(_WORKFLOW_DIR): + return (_REFERENCE_RE, _BARE_RE) + return (_REFERENCE_RE,) + + +def _candidates() -> list[tuple[str, list[str]]]: + """Every scanned file that did not opt out, as `(relative path, lines)`.""" + found: list[tuple[str, list[str]]] = [] for path_str in iter_non_ignored_files(): path = Path(path_str) if not _scanned(path): @@ -52,16 +79,41 @@ def missing_references() -> list[str]: continue if is_suppressed_in_head(lines, _NOCHECK_RULE): continue - rel = path.relative_to(PROJECT_ROOT).as_posix() + found.append((path.relative_to(PROJECT_ROOT).as_posix(), lines)) + return found + + +def missing_references() -> list[str]: + """Every `:: ` whose workflow is not on disk.""" + known = _known() + offenders: list[str] = [] + for rel, lines in _candidates(): + patterns = _patterns(PROJECT_ROOT / rel) for number, line in enumerate(lines, 1): + names = {name for pattern in patterns for name in pattern.findall(line)} offenders.extend( - f"{rel}:{number}: {name}" - for name in _REFERENCE_RE.findall(line) - if name not in known + f"{rel}:{number}: {name}" for name in sorted(names) if name not in known ) return offenders +def undispatchable_targets() -> list[str]: + """Every `gh workflow run` target that is absent or not manually dispatchable.""" + known = _known() + offenders: list[str] = [] + for rel, lines in _candidates(): + for number, line in enumerate(lines, 1): + for match in _DISPATCH_RE.finditer(line): + name = match.group("cli") or match.group("api") + if name not in known: + offenders.append(f"{rel}:{number}: {name} does not exist") + elif "workflow_dispatch" not in read_text(str(_WORKFLOW_DIR / name)): + offenders.append( + f"{rel}:{number}: {name} declares no workflow_dispatch trigger" + ) + return offenders + + class TestWorkflowReferences(unittest.TestCase): def test_referenced_workflows_exist(self) -> None: offenders = missing_references() @@ -73,6 +125,17 @@ class TestWorkflowReferences(unittest.TestCase): "that is still to be created:\n" + "\n".join(sorted(offenders)) ) + def test_dispatch_targets_are_dispatchable(self) -> None: + offenders = undispatchable_targets() + if offenders: + self.fail( + f"{len(offenders)} manual dispatch(es) name a workflow GitHub cannot " + "run. Both failures answer HTTP 422 rather than something the caller " + "can read: a stale filename resolves to the deleted workflow's " + "tombstone, and a reusable-only workflow refuses the event:\n" + + "\n".join(sorted(offenders)) + ) + if __name__ == "__main__": unittest.main() From 8f6c31056ba7cfd28fec8021979e9816181ca7d8 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 16:57:24 +0200 Subject: [PATCH 130/197] test(variants): refuse two variants that declare the same override Eight meta/variants.yml repeated their last entry verbatim. The six svc-* roles held the same tor.enabled override twice; fusiondirectory and phpldapadmin held a third entry identical to their second, nocheck comments included. The matrix plans a row per entry, assigns it its own axes and spends a deploy job on it, around a hundred minutes here. The repetition is invisible in the plan table, which lists rows by role#variant, so the two read as different rows and only their identical embeds and weight betray them. test_variants_distinct.py compares the parsed payload rather than the file text, so key order, indentation and comments do not hide a duplicate, and null normalises to {} because a bare list item and an explicit {} declare the same no-override entry. A role whose variants are told apart by something outside the entry opts the file out with `# nocheck: variants-duplicate` and a reason; no file needs it today. Every deletion is the trailing entry, so the surviving indices keep their meaning and nothing is renumbered. The vanishing tokens are svc-db-elasticsearch#1, svc-db-memcached#1, svc-db-rabbitmq#1, svc-db-redis#1, svc-db-typesense#1, svc-prx-openresty#1, web-app-fusiondirectory#2 and web-app-phpldapadmin#2; `git grep` against the commit finds no consumer naming any of them. Negative control executed: re-adding the redis duplicate fails the lint with "svc-db-redis: variants #0, #1 declare the same override", removing it passes. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-db-elasticsearch/meta/variants.yml | 3 - roles/svc-db-memcached/meta/variants.yml | 3 - roles/svc-db-rabbitmq/meta/variants.yml | 3 - roles/svc-db-redis/meta/variants.yml | 3 - roles/svc-db-typesense/meta/variants.yml | 3 - roles/svc-prx-openresty/meta/variants.yml | 3 - .../web-app-fusiondirectory/meta/variants.yml | 27 ------ roles/web-app-phpldapadmin/meta/variants.yml | 21 ----- .../roles/meta/test_variants_distinct.py | 83 +++++++++++++++++++ 9 files changed, 83 insertions(+), 66 deletions(-) create mode 100644 tests/lint/ansible/roles/meta/test_variants_distinct.py diff --git a/roles/svc-db-elasticsearch/meta/variants.yml b/roles/svc-db-elasticsearch/meta/variants.yml index 1576e2511e..a5e59537b6 100644 --- a/roles/svc-db-elasticsearch/meta/variants.yml +++ b/roles/svc-db-elasticsearch/meta/variants.yml @@ -2,6 +2,3 @@ - services: tor: enabled: true -- services: - tor: - enabled: true diff --git a/roles/svc-db-memcached/meta/variants.yml b/roles/svc-db-memcached/meta/variants.yml index 1576e2511e..a5e59537b6 100644 --- a/roles/svc-db-memcached/meta/variants.yml +++ b/roles/svc-db-memcached/meta/variants.yml @@ -2,6 +2,3 @@ - services: tor: enabled: true -- services: - tor: - enabled: true diff --git a/roles/svc-db-rabbitmq/meta/variants.yml b/roles/svc-db-rabbitmq/meta/variants.yml index 1576e2511e..a5e59537b6 100644 --- a/roles/svc-db-rabbitmq/meta/variants.yml +++ b/roles/svc-db-rabbitmq/meta/variants.yml @@ -2,6 +2,3 @@ - services: tor: enabled: true -- services: - tor: - enabled: true diff --git a/roles/svc-db-redis/meta/variants.yml b/roles/svc-db-redis/meta/variants.yml index 1576e2511e..a5e59537b6 100644 --- a/roles/svc-db-redis/meta/variants.yml +++ b/roles/svc-db-redis/meta/variants.yml @@ -2,6 +2,3 @@ - services: tor: enabled: true -- services: - tor: - enabled: true diff --git a/roles/svc-db-typesense/meta/variants.yml b/roles/svc-db-typesense/meta/variants.yml index 1576e2511e..a5e59537b6 100644 --- a/roles/svc-db-typesense/meta/variants.yml +++ b/roles/svc-db-typesense/meta/variants.yml @@ -2,6 +2,3 @@ - services: tor: enabled: true -- services: - tor: - enabled: true diff --git a/roles/svc-prx-openresty/meta/variants.yml b/roles/svc-prx-openresty/meta/variants.yml index 1576e2511e..a5e59537b6 100644 --- a/roles/svc-prx-openresty/meta/variants.yml +++ b/roles/svc-prx-openresty/meta/variants.yml @@ -2,6 +2,3 @@ - services: tor: enabled: true -- services: - tor: - enabled: true diff --git a/roles/web-app-fusiondirectory/meta/variants.yml b/roles/web-app-fusiondirectory/meta/variants.yml index bf19c2930c..ff10fee77b 100644 --- a/roles/web-app-fusiondirectory/meta/variants.yml +++ b/roles/web-app-fusiondirectory/meta/variants.yml @@ -60,30 +60,3 @@ shared: false # nocheck: variants-static-override backup shared stays literally true tor: enabled: true -- services: - container_backup: - enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true - sso: - enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true - css: - enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true - dashboard: - enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true - email: - enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true - logout: - enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true - matomo: - enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true - prometheus: - enabled: false - shared: false # nocheck: variants-static-override backup shared stays literally true - tor: - enabled: true diff --git a/roles/web-app-phpldapadmin/meta/variants.yml b/roles/web-app-phpldapadmin/meta/variants.yml index f81db4c599..c986849575 100644 --- a/roles/web-app-phpldapadmin/meta/variants.yml +++ b/roles/web-app-phpldapadmin/meta/variants.yml @@ -52,24 +52,3 @@ shared: false tor: enabled: true -- services: - sso: - enabled: false - shared: false - css: - enabled: false - shared: false - dashboard: - enabled: false - shared: false - logout: - enabled: false - shared: false - matomo: - enabled: false - shared: false - prometheus: - enabled: false - shared: false - tor: - enabled: true diff --git a/tests/lint/ansible/roles/meta/test_variants_distinct.py b/tests/lint/ansible/roles/meta/test_variants_distinct.py new file mode 100644 index 0000000000..3dc998b5aa --- /dev/null +++ b/tests/lint/ansible/roles/meta/test_variants_distinct.py @@ -0,0 +1,83 @@ +"""No two entries of a ``meta/variants.yml`` may declare the same override. + +A variant exists to deploy a shape the other entries do not. Two entries with +the same content produce the same rendered payload, so the matrix plans them as +separate rows, assigns each its own axes and spends a deploy job on each. The +measured cost of one such job in this repository's CI is around one hundred +minutes, and the second one proves nothing the first did not. + +The duplicate is also invisible in the plan table, which lists rows by +``role#variant``: the two read as different rows and only their identical +``embeds`` and ``weight`` betray them. + +Comparison is on the parsed payload, not the file text, so two entries that +differ in key order, indentation or comments still count as the same. ``null`` +normalises to ``{}`` the way the loader treats it, because a bare ``- `` list +item and an explicit ``- {}`` declare the same no-override entry. + +Per-file opt-out +================ +``# nocheck: variants-duplicate`` anywhere in the file, with a reason naming +what the repetition buys. Reserve it for a role whose variants are told apart +by something outside the entry, and say what that is; the default path is to +delete the entry, because the axes a duplicate would have covered are assigned +per row rather than declared per variant. +""" + +from __future__ import annotations + +import json +import unittest +from collections import defaultdict + +from utils.annotations.suppress import is_suppressed_anywhere +from utils.cache.files import read_text +from utils.cache.yaml import load_yaml_any +from utils.roles.mapping import ROLE_FILE_META_VARIANTS + +from . import PROJECT_ROOT + +RULE = "variants-duplicate" + + +def _fingerprint(entry: object) -> str: + """The entry's content, independent of key order and of ``null`` vs ``{}``.""" + return json.dumps(entry if entry is not None else {}, sort_keys=True, default=str) + + +class TestVariantsDistinct(unittest.TestCase): + def test_no_two_variants_declare_the_same_override(self) -> None: + offenders = [] + for variants_file in sorted( + (PROJECT_ROOT / "roles").glob(f"*/{ROLE_FILE_META_VARIANTS}") + ): + entries = load_yaml_any(str(variants_file), default_if_missing=[]) + if not isinstance(entries, list): + continue + if is_suppressed_anywhere(read_text(str(variants_file)).splitlines(), RULE): + continue + + seen: dict[str, list[int]] = defaultdict(list) + for index, entry in enumerate(entries): + seen[_fingerprint(entry)].append(index) + + role = variants_file.parent.parent.name + for indices in seen.values(): + if len(indices) > 1: + joined = ", ".join(f"#{index}" for index in indices) + offenders.append( + f"{role}: variants {joined} declare the same override" + ) + + self.assertEqual( + [], + offenders, + "A duplicated variant entry costs a deploy job and proves nothing the " + "entry it repeats did not. Delete the repetition, give it the override " + f"that makes it a distinct shape, or mark the file `# nocheck: {RULE}` " + "with a reason:\n " + "\n ".join(offenders), + ) + + +if __name__ == "__main__": + unittest.main() From e7ade5fb2729e9dd081595bce43a807a09a6b712 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 18 Sep 2026 16:59:04 +0200 Subject: [PATCH 131/197] docs(claude): make an agent pin a green run to the line that ran MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A job conclusion aggregates "passed" and "did not run" into one colour, and reading the colour turns an unexercised change into a claimed proof. Measured on 2026-09-18 in run 35330645276: jobs 105558220731 (Fider#1) and 105558220741 (Fider#0) are both green. Fider#1 logs "- 4 tests/playwright.spec.js:56:1 > fider: admin sso login", where the leading dash is Playwright's skipped marker, because that variant has SSO off and the spec opens with test.skip(!oidcEnabled, ...). Only Fider#0 logs the same spec as "โœ“ ... (1.7s)" and actually exercised the fix. The second rule covers the fork. Tags do not follow a fork and repository variables differ, so a step gated on either never runs there: the release cron is 25 of 25 green on the fork and 25 of 25 red upstream, because the fork holds 2 tags against upstream's 60+ and `if: steps.pick.outputs.tag != ''` can never become true. In this checkout origin is infinito-nexus/core with gh-resolved=base while pushDefault is the fork, so a bare `gh run list` shows upstream while push runs land on the fork. Neither rule is enforceable by a check, because "which line proves this fix" is not machine-decidable. They are written where the agent reads them and recorded in agent memory so they survive the file. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index c468b65997..7c96f38bcc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,11 @@ Anything else โ€” restating code, section banners, "Note that โ€ฆ", step narrati - Default the filename to a meaningful slug + monotonically increasing index (`/tmp/make-test--.log`, `/tmp/act--.log`) so you can compare runs. - Tell the operator the exact `tail -f /tmp/.log` command for background runs, per the existing rule. +## CI evidence ๐Ÿ“Š + +- A job's green conclusion is **NOT** evidence for a specific change. Before citing a run as proof, you MUST locate the line in that job's log where the change executed: a Playwright spec's `โœ“` (a leading `-` means the spec was skipped), the env var inside the container, or the command text of an `if:`-gated step. If the string is absent, report the run as silent on the change, not as supporting it. +- A green run on the fork does not speak for `infinito-nexus/core`. Tags do not follow a fork, and repository variables differ, so a step gated on either never runs there. Check the repository whose failure you are claiming to have fixed. + ## Pushing ๐Ÿšข - You MUST NOT push, directly or through wrappers that push implicitly. From afe2bed2a9a71f1a36a8bf1efd021541d382b58e Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 03:19:43 +0000 Subject: [PATCH 132/197] update(images): update Docker image versions (svc-ai-ollama) --- roles/svc-ai-ollama/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/svc-ai-ollama/meta/services.yml b/roles/svc-ai-ollama/meta/services.yml index 63e0060d7f..6b8ef95ccd 100644 --- a/roles/svc-ai-ollama/meta/services.yml +++ b/roles/svc-ai-ollama/meta/services.yml @@ -13,7 +13,7 @@ ollama: backup: no_stop_required: true image: ollama/ollama - version: 0.34.0 + version: 0.34.2 name: ollama min_storage: 3473MB cpus: "4.0" From 64267bea15c86877f3562dd1d05ea95f3f8fbf6e Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 03:20:00 +0000 Subject: [PATCH 133/197] update(images): update Docker image versions (web-app-baserow) --- roles/web-app-baserow/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-baserow/meta/services.yml b/roles/web-app-baserow/meta/services.yml index 70f1f37967..e87651f484 100644 --- a/roles/web-app-baserow/meta/services.yml +++ b/roles/web-app-baserow/meta/services.yml @@ -62,7 +62,7 @@ baserow: backup: no_stop_required: true image: baserow/baserow - version: 2.3.3 + version: 2.3.4 custom: true name: baserow min_storage: 785MB From b642e16e626f4eed3c23f3b8652e4551ffe2fd14 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 03:20:10 +0000 Subject: [PATCH 134/197] update(images): update Docker image versions (web-app-funkwhale) --- roles/web-app-funkwhale/meta/services.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/web-app-funkwhale/meta/services.yml b/roles/web-app-funkwhale/meta/services.yml index d102779090..92cd7265dc 100644 --- a/roles/web-app-funkwhale/meta/services.yml +++ b/roles/web-app-funkwhale/meta/services.yml @@ -11,7 +11,7 @@ funkwhale: # nocheck: default-placement-manager embedded typesense index; node-local in swarm placement: manager image: funkwhale/front - version: 2.0.10 + version: 2.0.11 min_storage: 224MB ports: internal: @@ -35,7 +35,7 @@ api: flavor: curl path: api/v1/instance/nodeinfo/2.0 image: funkwhale/api - version: 2.0.10 + version: 2.0.11 min_storage: 588MB ports: internal: From 147dc2707f89d40bc761de8eb38e0ca72c749d34 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 03:20:18 +0000 Subject: [PATCH 135/197] update(images): update Docker image versions (web-app-keycloak) --- roles/web-app-keycloak/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-keycloak/meta/services.yml b/roles/web-app-keycloak/meta/services.yml index fe60b74828..b43b4a9c8a 100644 --- a/roles/web-app-keycloak/meta/services.yml +++ b/roles/web-app-keycloak/meta/services.yml @@ -51,7 +51,7 @@ keycloak: shared: true provides: sso image: quay.io/keycloak/keycloak - version: "26.7.3" + version: "26.7.4" name: keycloak min_storage: 468MB cpus: "2.0" From 80bd969b4e97b9aec593c4cf855c7ed14372e09f Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 03:20:32 +0000 Subject: [PATCH 136/197] update(images): update Docker image versions (web-app-opentalk) --- roles/web-app-opentalk/meta/services.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/web-app-opentalk/meta/services.yml b/roles/web-app-opentalk/meta/services.yml index 352669e2c2..d5d2472389 100644 --- a/roles/web-app-opentalk/meta/services.yml +++ b/roles/web-app-opentalk/meta/services.yml @@ -96,7 +96,7 @@ controller: livekit: name: opentalk-livekit image: livekit/livekit-server - version: v1.13.6 + version: v1.13.7 min_storage: 238MB cpus: "2.0" mem_reservation: 512m @@ -129,7 +129,7 @@ seaweedfs: rabbit: name: opentalk-rabbit image: rabbitmq - version: 4.3.5 + version: 4.3.6 min_storage: 314MB cpus: "0.5" mem_reservation: 256m From cc7e774a892a14796099b8d82c60b4945cad1950 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 03:20:36 +0000 Subject: [PATCH 137/197] update(images): update Docker image versions (web-app-peertube) --- roles/web-app-peertube/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-peertube/meta/services.yml b/roles/web-app-peertube/meta/services.yml index 37a0a9535a..cdfcd1f277 100644 --- a/roles/web-app-peertube/meta/services.yml +++ b/roles/web-app-peertube/meta/services.yml @@ -61,7 +61,7 @@ peertube: enabled: true # nocheck: dynamic-flag shared: true name: peertube - version: v8.2.4 + version: v8.3.0 image: chocobozzz/peertube min_storage: 841MB live: From e0f5295ccfa98c947984a2122a69d45826fb3601 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 03:20:44 +0000 Subject: [PATCH 138/197] update(images): update Docker image versions (web-app-prometheus) --- roles/web-app-prometheus/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-prometheus/meta/services.yml b/roles/web-app-prometheus/meta/services.yml index bd6b8702a8..0cf9196041 100644 --- a/roles/web-app-prometheus/meta/services.yml +++ b/roles/web-app-prometheus/meta/services.yml @@ -79,7 +79,7 @@ alertmanager: flavor: wget path: -/healthy image: prom/alertmanager # nocheck: volumes-path-single-spot: the repository path of an image name, not a mount target - version: v0.34.0 + version: v0.34.1 name: prometheus-alertmanager cpus: "0.25" mem_reservation: 64m From ade1b1186141a12b2be2dbdfa09fcd04ade2b433 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 03:20:52 +0000 Subject: [PATCH 139/197] update(images): update Docker image versions (web-app-socialhome) --- roles/web-app-socialhome/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-socialhome/meta/services.yml b/roles/web-app-socialhome/meta/services.yml index 388c650bc1..59ddf807e3 100644 --- a/roles/web-app-socialhome/meta/services.yml +++ b/roles/web-app-socialhome/meta/services.yml @@ -13,7 +13,7 @@ socialhome: # nocheck: default-placement-manager embedded sqlite in a node-local volume; no clustered storage placement: manager image: ghcr.io/social-home-io/socialhome - version: "2026.9.11" + version: "2026.9.18" ports: local: http: 8089 From 4e7f14a29d57b5f5a57a3bf1e8deea39ba574791 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 05:46:58 +0200 Subject: [PATCH 140/197] fix(lmstudio): let the model import answer its prompt and cross the volume The seed path added in ee582762b0 fetched the model and never handed it over. Two independent faults sat on top of each other; the second only became visible once the first was gone. First, the prompt. `lms import` keeps asking "Do you wish to continue? (Y/n)" even with `--yes`, which suppresses only the warning about moving the file. import.sh ran it through `container exec` without `-i`, so the question met a closed stdin and the script died with 130, printing neither PRESENT:, ABSENT: nor IMPORTED:. The sibling call in tasks/utils/preload.yml already passes `-i`. Second, the move. `lms import` renames by default. `container cp` stages the file in /tmp while MODELS_DIR lives on the mounted volume, so rename(2) crosses a device boundary and cannot work: EXDEV: cross-device link not permitted, rename '/tmp/...gguf' -> '/root/.lmstudio/models/lmstudio-community/...' Hence `--copy`, and hence the `rm` that clears the staged file copying leaves behind. That path could therefore never have succeeded, which is why every CI job since 2026-09-17 fetched 531068224 bytes, verified their sha256 and threw them away. Dropping `failed_when: false` is what makes the next failure visible: it reported the abort as `ok` in 1.40s. The task name loses "best effort" for the same reason. Measured in a local compose deploy: IMPORTED:Qwen2.5-0.5B-Instruct-Q8_0.gguf, then `lms get` returns rc 0 after 0.895s instead of a 60s network abort followed by three 3600s timeout kills, both play recaps failed=0. The short-circuits still hold: on the second pass the node cache check costs 0.64s instead of 145.24s and import.sh prints PRESENT: before it copies anything. Residual: rc 130 was measured on compose only. Some of the eight matrix rows that pull svc-ai-lmstudio take the async swarm path, where the import is unverified. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-lmstudio/files/shell/import.sh | 3 ++- roles/svc-ai-lmstudio/tasks/utils/seed.yml | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/svc-ai-lmstudio/files/shell/import.sh b/roles/svc-ai-lmstudio/files/shell/import.sh index 98667d5d02..5e9e848d40 100644 --- a/roles/svc-ai-lmstudio/files/shell/import.sh +++ b/roles/svc-ai-lmstudio/files/shell/import.sh @@ -29,5 +29,6 @@ fi # nocheck: container-cp - MODEL_FILE is fetched on the node this script runs on container cp "${MODEL_FILE}" "${ADDRESS}:/tmp/${BASE}" -container exec "$ADDRESS" lms import "/tmp/${BASE}" --yes --user-repo "${MODEL_USER_REPO}" +printf 'y\n' | container exec -i "$ADDRESS" lms import "/tmp/${BASE}" --yes --copy --user-repo "${MODEL_USER_REPO}" +container exec "$ADDRESS" rm -f "/tmp/${BASE}" echo "IMPORTED:${BASE}" diff --git a/roles/svc-ai-lmstudio/tasks/utils/seed.yml b/roles/svc-ai-lmstudio/tasks/utils/seed.yml index ec28cea89e..09eb7f45e6 100644 --- a/roles/svc-ai-lmstudio/tasks/utils/seed.yml +++ b/roles/svc-ai-lmstudio/tasks/utils/seed.yml @@ -18,7 +18,7 @@ changed_when: false failed_when: false -- name: "๐Ÿ“‹ Hand a warmed model to LM Studio, best effort" +- name: "๐Ÿ“‹ Hand the warmed model to LM Studio" ansible.builtin.script: shell/import.sh environment: LMSTUDIO_LOCAL_CID: "{{ LMSTUDIO_LOCAL_CID }}" @@ -30,4 +30,3 @@ loop_var: model label: "{{ model.file }}" changed_when: false - failed_when: false From 9d940f40404ac479aafc9de4c156f11b260266ff Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 05:55:51 +0200 Subject: [PATCH 141/197] fix(nextcloud): install onlyoffice from a release that upstream published MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ONLYOFFICE listed onlyoffice 10.2.0 in the Nextcloud appstore on 2026-09-18 at 10:16:31Z and never published the matching GitHub artifact; the tag itself does not exist. `occ app:install` takes the newest compatible release, so it dies on a 404 that no retry can heal. CI proved that twice independently: upstream run 35329488812 and fork run 35330645276 both fail Nextcloud#4 at that task after six attempts, on the same commit, for the same reason. The fault had three layers, each visible only once the one above it was gone. Install. meta/addons/onlyoffice.yml pins `config.archive` to v10.1.2, whose artifact answers 200 and which the appstore also lists for this server major. 01_install.yml stages that archive on the node and unpacks it into custom_apps through a stdin pipe: the webroot is a named volume, so the node can only reach the app directory through the container, and piping avoids both `container cp` and a cleanup step that would have to run as a user which does not own the file. Enable and configure needed no change at all. With the release already on disk `occ app:install` reports "already installed", which nextcloud_install_status has always mapped to runnable, so 02_enable.yml and 03_configure.yml proceed as they always did. That is why this commit adds no branch to the plugin pipeline. Update. `occ app:update --all` then dragged the pinned app back to the broken 10.2.0 and killed the deploy. occ offers no exclusion flag, but it does offer `--showonly` and single app ids, so a pinned app is simply not updated. The old `--all` call stays byte-identical and now carries a `when`: with nothing pinned, thirteen of the fourteen variants take exactly the path they took before. This layer cannot appear in CI at all, because it needs a second deploy with the app already installed, and the install always died first. The classifier test is inverted rather than deleted: test_an_advertised_release_whose_artifact_is_gone_stays_fatal pins the decision that a dead artifact fails loudly instead of degrading to a silent skip. Verified by a local compose deploy of variant 4: ok=2508 changed=149 failed=0 and `โœ“ 36 tests/addons/onlyoffice.spec.js:12:1`, which drives the whole JWT-authenticated round trip to the document server over its onion address, not merely an enabled plugin. In the live container beforehand: `occ app:enable` returned "onlyoffice 10.1.2 enabled", `occ app:install` returned "already installed", and `occ app:update onlyoffice` reproduced the 404 that the pin now avoids. The loop expression was wrong before this commit and is fixed here: Jinja reads '\1' as an octal escape, so the backreference produced chr(1) instead of an app id. Evaluated against ansible-core's own filters over four shapes, the tail is now cut instead of captured, which removes the escaping hazard entirely. occ output is not localised: with default_language=de and a de_DE.UTF-8 locale the strings are unchanged, so parsing them is as safe as the five markers the role already relied on. Residual: the per-app update branch has never run with a real item inside a deploy. Its chain and `occ app:update ` are each verified separately. Co-Authored-By: Claude Opus 5 (1M context) --- .../meta/addons/onlyoffice.yml | 1 + .../tasks/02_manager_ops/03_upgrade.yml | 24 +++++++++++++++++++ .../02_manager_ops/06_plugin/01_install.yml | 18 ++++++++++++++ roles/web-app-nextcloud/vars/main.yml | 7 ++++++ .../filter_plugins/test_nextcloud_install.py | 10 ++++++++ 5 files changed, 60 insertions(+) diff --git a/roles/web-app-nextcloud/meta/addons/onlyoffice.yml b/roles/web-app-nextcloud/meta/addons/onlyoffice.yml index c319f03d69..00a09a6158 100644 --- a/roles/web-app-nextcloud/meta/addons/onlyoffice.yml +++ b/roles/web-app-nextcloud/meta/addons/onlyoffice.yml @@ -10,6 +10,7 @@ update: catalog: nextcloud-appstore upstream_id: onlyoffice config: + archive: "https://github.com/ONLYOFFICE/onlyoffice-nextcloud/releases/download/v10.1.2/onlyoffice.tar.gz" incompatible_plugins: - richdocuments - fileslibreofficeedit diff --git a/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade.yml b/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade.yml index 9b7abf8f3b..c4960649e0 100644 --- a/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade.yml +++ b/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade.yml @@ -22,6 +22,30 @@ retries: 2 delay: 3 until: occ_app_update.rc == 0 + when: NEXTCLOUD_PINNED_PLUGINS | length == 0 + +- name: ๐Ÿ” Nextcloud | List the apps the appstore offers an update for + shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} app:update --all --showonly" + register: occ_app_updatable + changed_when: false + retries: 2 + delay: 3 + until: occ_app_updatable.rc == 0 + when: NEXTCLOUD_PINNED_PLUGINS | length > 0 + +- name: ๐Ÿ”„ Nextcloud | App update, leaving the pinned releases alone + shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} app:update {{ item | quote }}" + register: occ_app_update_unpinned + changed_when: "'No apps found for update' not in occ_app_update_unpinned.stdout" + retries: 2 + delay: 3 + until: occ_app_update_unpinned.rc == 0 + when: NEXTCLOUD_PINNED_PLUGINS | length > 0 + loop: >- + {{ occ_app_updatable.stdout_lines | default([]) + | select('search', ' new version available') + | map('regex_replace', ' new version available.*$', '') + | difference(NEXTCLOUD_PINNED_PLUGINS) }} - name: โž• Nextcloud | Add missing columns shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} db:add-missing-columns" diff --git a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/01_install.yml b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/01_install.yml index c92cb89644..a9810511df 100644 --- a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/01_install.yml +++ b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/01_install.yml @@ -4,6 +4,24 @@ ansible.builtin.set_fact: plugin_incompatible: "{{ plugin_value.config.incompatible_plugins | default(plugin_value.incompatible_plugins | default([])) }}" plugin_mandatory: "{{ plugin_value.config.mandatory | default(plugin_value.mandatory | default(false)) }}" + plugin_archive: "{{ plugin_value.config.archive | default(plugin_value.archive | default('')) }}" + changed_when: false + +- name: "โฌ‡๏ธ Stage the pinned {{ plugin_key }} release on the node" + get_url_retry: + url: "{{ plugin_archive }}" + dest: "/tmp/nextcloud-plugin-{{ plugin_key }}.tar.gz" + mode: "0644" + when: plugin_archive | length > 0 + +- name: "๐Ÿ“ฆ Unpack the pinned {{ plugin_key }} release so occ finds it installed" + ansible.builtin.shell: >- + cat /tmp/nextcloud-plugin-{{ plugin_key }}.tar.gz + | {{ NEXTCLOUD_DOCKER_EXEC_INTERACTIVE }} + tar -xzf - -C {{ NEXTCLOUD_APPS_DIRECTORY | quote }} + args: + executable: /bin/bash + when: plugin_archive | length > 0 changed_when: false - name: "๐Ÿ”ด Disable incompatible plugins for {{ plugin_key }} (batched shell, no async)" diff --git a/roles/web-app-nextcloud/vars/main.yml b/roles/web-app-nextcloud/vars/main.yml index cee723dad1..f0cdb2c8c5 100644 --- a/roles/web-app-nextcloud/vars/main.yml +++ b/roles/web-app-nextcloud/vars/main.yml @@ -18,6 +18,11 @@ NEXTCLOUD_OIDC_HOST: "{{ lookup('tls', 'web-app-keycloak', 'domain') }}" NEXTCLOUD_TOR_SOCKS_PROXY: "{{ lookup('tor_socks_proxy') }}" NEXTCLOUD_PLUGIN_ITEMS: "{{ lookup('config', application_id, 'addons') | dict2items }}" +NEXTCLOUD_PINNED_PLUGINS: >- + {{ NEXTCLOUD_PLUGIN_ITEMS + | selectattr('value.config.archive', 'defined') + | selectattr('value.enabled', 'true') + | map(attribute='key') | list }} NEXTCLOUD_PLUGINS_ENABLED: "{{ lookup('config', application_id, 'services.nextcloud.plugins_enabled') }}" NEXTCLOUD_USER_LDAP_ENABLED: "{{ lookup('config', application_id, 'addons.user_ldap.enabled') }}" NEXTCLOUD_LDAP_CONFIG_PREFIX: "s01" @@ -221,7 +226,9 @@ NEXTCLOUD_DOCKER_USER: "www-data" NEXTCLOUD_FPM_PORT: "{{ lookup('config', application_id, 'services.nextcloud.ports.internal.fpm') }}" NEXTCLOUD_INTERNAL_OCC_COMMAND: "{{ [ NEXTCLOUD_DOCKER_WORK_DIRECTORY, 'occ'] | path_join }}" +NEXTCLOUD_APPS_DIRECTORY: "{{ [ NEXTCLOUD_DOCKER_WORK_DIRECTORY, 'custom_apps'] | path_join }}" NEXTCLOUD_DOCKER_EXEC: "container exec -u {{ NEXTCLOUD_DOCKER_USER }} {{ NEXTCLOUD_LOCAL_CID | quote }}" # nocheck: container-exec-resolver NEXTCLOUD_LOCAL_CID resolved at delegate_to runtime; container_address would return a manager-only subshell NEXTCLOUD_DOCKER_EXEC_OCC: "{{ NEXTCLOUD_DOCKER_EXEC }} {{ NEXTCLOUD_INTERNAL_OCC_COMMAND }}" +NEXTCLOUD_DOCKER_EXEC_INTERACTIVE: "container exec -i -u {{ NEXTCLOUD_DOCKER_USER }} {{ NEXTCLOUD_LOCAL_CID | quote }}" # nocheck: container-exec-resolver NEXTCLOUD_LOCAL_CID resolved at delegate_to runtime, as for NEXTCLOUD_DOCKER_EXEC NEXTCLOUD_REDIS_CONTAINER_ADDRESS: "{{ lookup('container_address', application_id, 'redis') }}" diff --git a/tests/unit/python/roles/web-app-nextcloud/filter_plugins/test_nextcloud_install.py b/tests/unit/python/roles/web-app-nextcloud/filter_plugins/test_nextcloud_install.py index b633104387..3b902b60f8 100644 --- a/tests/unit/python/roles/web-app-nextcloud/filter_plugins/test_nextcloud_install.py +++ b/tests/unit/python/roles/web-app-nextcloud/filter_plugins/test_nextcloud_install.py @@ -12,6 +12,11 @@ SOCKS_ERROR = ( "for https://garm3.nextcloud.com/api/v1/apps.json" ) TIMEOUT_ERROR = "cURL error 28: Connection timed out after 120002 milliseconds" +ARTIFACT_404 = ( + "Error: Client error: `GET https://github.com/ONLYOFFICE/onlyoffice-nextcloud" + "/releases/download/v10.2.0/onlyoffice.tar.gz` resulted in a " + "`404 Not Found` response:\nNot Found" +) def _load_module(rel_path: str, name: str): @@ -56,6 +61,11 @@ class TestNextcloudInstallStatus(unittest.TestCase): self.assertFalse(status["unavailable"]) self.assertFalse(_tolerated(status)) + def test_an_advertised_release_whose_artifact_is_gone_stays_fatal(self): + status = _status(stdout=ARTIFACT_404) + self.assertFalse(status["unavailable"]) + self.assertFalse(_tolerated(status)) + def test_an_incompatible_plugin_stays_terminal_and_not_runnable(self): status = _status( stdout="deck is not compatible with this version of the server" From 673de05a60f10a22c81a88e5815233278bc34a5a Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 06:00:56 +0200 Subject: [PATCH 142/197] fix(inventory): write only the service toggles a role's contract declares `disable=` wrote both `enabled: false` and `shared: false` for every role that declares the service, but a role qualifies with any one of `enabled`, `shared` or `provides`. compare_application_keys rejects any host_vars key the role contract does not carry, so a role declaring only `enabled` came back as a validation error and the deploy never reached Ansible: Missing default for web-svc-cdn: services.dashboard.shared Missing default for web-svc-mirror: services.dashboard.shared [ERROR] Inventory validation failed. Aborting deployment. _declared_toggles reads the role's meta/services.yml and writes only what is there; a role carrying neither toggle is skipped instead of gaining an empty block. The module already held this principle: find_roles_with_service excludes pure metadata entries so `disable` "doesn't add bogus enabled: false overrides", and this applies the same rule per flag rather than per role. Delta measured over the disable list the compose iteration procedure prescribes (matomo, dashboard, prometheus, email, css): 407 of 409 role/service pairs are untouched because they declare both toggles, exactly 2 stop receiving a fabricated key, and 0 fall out of the disable entirely. The 2 are the pairs the validator rejected. Across all 266 roles with a meta/services.yml, 1224 service blocks declare `enabled`, 1190 declare `shared`, 34 declare `enabled` alone and none declares `provides` alone, so nothing loses its disable. Proven by the deploy sequence rather than by a test: the first local deploy aborted at inventory validation, the second passed into the Ansible run with the same disable list. Two regression tests cover both shapes, and the negative control holds: hard-coding ["enabled", "shared"] fails the new test. Without this, `disable=` is unusable in every local deploy, which the iteration procedure requires on every run. Residual: the generator was exercised through compose-deploy only, not through roundtrip or swarm-zombie, which reach the same function by other call paths. Co-Authored-By: Claude Opus 5 (1M context) --- .../inventory/provision/services_disabler.py | 29 ++++++++++++++---- .../provision/test_services_disabler.py | 30 +++++++++++++++++++ 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/cli/administration/inventory/provision/services_disabler.py b/cli/administration/inventory/provision/services_disabler.py index 7f01e59260..33b4ac0cb1 100644 --- a/cli/administration/inventory/provision/services_disabler.py +++ b/cli/administration/inventory/provision/services_disabler.py @@ -86,6 +86,21 @@ def find_roles_with_service(service_name: str, roles_dir: Path) -> set[str]: return role_ids +def _declared_toggles(app_id: str, svc_name: str, roles_dir: Path) -> list[str]: + """The toggles this role's contract carries for this service, in file order. + + `compare_application_keys` rejects any host_vars key the role contract does + not declare, so writing a toggle the role never had turns a `disable=` run + into an inventory the validator refuses. 34 service blocks declare + `enabled` without `shared`. + """ + services = _load_yaml_mapping_tolerant(roles_dir / app_id / ROLE_FILE_META_SERVICES) + entry = services.get(svc_name) if services else None + if not isinstance(entry, dict): + return [] + return [flag for flag in ("enabled", "shared") if flag in entry] + + def find_provider_roles(services: list[str], roles_dir: Path) -> dict[str, str]: """ Return mapping of service_name -> application_id for the provider role of each @@ -204,17 +219,19 @@ def apply_services_disabled( svc_map = CommentedMap() app_data["services"] = svc_map + flags = _declared_toggles(app_id, svc_name, roles_dir) + if not flags: + continue + svc = svc_map.get(svc_name) if not isinstance(svc, CommentedMap): svc = CommentedMap() svc_map[svc_name] = svc - svc["enabled"] = False - svc["shared"] = False + for flag in flags: + svc[flag] = False changed = True - print( - f"[INFO] disable: {app_id}.services.{svc_name} " - "โ†’ enabled=false, shared=false" - ) + written = ", ".join(f"{flag}=false" for flag in flags) + print(f"[INFO] disable: {app_id}.services.{svc_name} โ†’ {written}") if changed: with host_vars_file.open("w", encoding="utf-8") as f: diff --git a/tests/unit/python/cli/administration/inventory/provision/test_services_disabler.py b/tests/unit/python/cli/administration/inventory/provision/test_services_disabler.py index 3b6d865ff2..59155d1424 100644 --- a/tests/unit/python/cli/administration/inventory/provision/test_services_disabler.py +++ b/tests/unit/python/cli/administration/inventory/provision/test_services_disabler.py @@ -188,6 +188,36 @@ class TestApplyServicesDisabled(unittest.TestCase): dump_yaml(role_dir / ROLE_FILE_META_SERVICES, services) dump_yaml(role_dir / ROLE_FILE_VARS_MAIN, {"application_id": role_name}) + def test_a_role_without_shared_in_its_contract_does_not_get_one(self): + self._write_host_vars( + {"applications": {"web-svc-cdn": {"services": {"dashboard": {}}}}} + ) + self._make_role("web-svc-cdn", {"dashboard": {"enabled": False}}) + + apply_services_disabled(self.host_vars, ["dashboard"], self.roles_dir) + + service = self._read_host_vars()["applications"]["web-svc-cdn"]["services"][ + "dashboard" + ] + self.assertEqual(False, service["enabled"]) + self.assertNotIn("shared", service) + + def test_a_role_declaring_both_toggles_still_gets_both(self): + self._write_host_vars( + {"applications": {"web-app-nextcloud": {"services": {"dashboard": {}}}}} + ) + self._make_role( + "web-app-nextcloud", {"dashboard": {"enabled": True, "shared": True}} + ) + + apply_services_disabled(self.host_vars, ["dashboard"], self.roles_dir) + + service = self._read_host_vars()["applications"]["web-app-nextcloud"][ + "services" + ]["dashboard"] + self.assertEqual(False, service["enabled"]) + self.assertEqual(False, service["shared"]) + def test_disables_service_in_host_vars_and_removes_from_inventory(self): self._write_host_vars( { From 30c3d699fdc4fe8e38196edbf1fba6022b7f43cb Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 06:03:40 +0200 Subject: [PATCH 143/197] docs: name the workflows the prefixing sweep left behind 71888d86af moved twelve workflow files behind trigger prefixes and updated the references GitHub validates. Three mentions in prose kept the old names: an echo line, a file header and a lint's failure message. The same commit already fixed the REST dispatch URL two lines below the echo it missed. roles/svc-runner/files/test/external.sh:36 test-runner-smoke.yml scripts/github/runner/cleanup_docker.sh:4 images-build-ci.yml tests/lint/repository/test_distros_spot.py:116 test-workspace.yml None of them executes; the workflow-reference lint deliberately does not reach them, because outside .github/workflows a bare *.yml collides with services.yml, compose.yml and variants.yml. A repo-wide grep over all eleven legacy names now returns nothing. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-runner/files/test/external.sh | 2 +- scripts/github/runner/cleanup_docker.sh | 2 +- tests/lint/repository/test_distros_spot.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/svc-runner/files/test/external.sh b/roles/svc-runner/files/test/external.sh index 28378ec774..6a36661122 100644 --- a/roles/svc-runner/files/test/external.sh +++ b/roles/svc-runner/files/test/external.sh @@ -33,7 +33,7 @@ else echo "OK: ${online_count} runner(s) online on GitHub (${RUNNER_GITHUB_OWNER}/${RUNNER_GITHUB_REPO})" fi -echo "Dispatching test-runner-smoke.yml against ref=${RUNNER_GIT_REF}..." +echo "Dispatching call-test-runner-smoke.yml against ref=${RUNNER_GIT_REF}..." dispatch_time=$(date -u +%Y-%m-%dT%H:%M:%SZ) dispatch_http=$(${CURL} -sf -o /dev/null -w "%{http_code}" \ diff --git a/scripts/github/runner/cleanup_docker.sh b/scripts/github/runner/cleanup_docker.sh index 0a210d463b..85a45dc965 100755 --- a/scripts/github/runner/cleanup_docker.sh +++ b/scripts/github/runner/cleanup_docker.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # # Cleanup pass for the Docker / buildx state on a GitHub-hosted runner. -# Used by images-build-ci.yml as the always-on tail of the build job to +# Used by call-images-build-ci.yml as the always-on tail of the build job to # free disk before the runner is reused (or recycled). set -euo pipefail diff --git a/tests/lint/repository/test_distros_spot.py b/tests/lint/repository/test_distros_spot.py index fe380047fc..5f58618776 100644 --- a/tests/lint/repository/test_distros_spot.py +++ b/tests/lint/repository/test_distros_spot.py @@ -113,7 +113,7 @@ class TestDistrosSpot(unittest.TestCase): ref = _MATRIX_REF_RE.match(str(axis)) self.assertIsNotNone( ref, - f"test-workspace.yml must derive its matrix from {FILE_META_DISTROS} " + f"call-test-workspace.yml must derive its matrix from {FILE_META_DISTROS} " f"via fromJson(needs..outputs.), got {axis!r}.", ) resolver_id, output_name = ref.groups() From 835584e2c924da0d2fbb0c75b62e704ad75d2531 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 06:30:36 +0200 Subject: [PATCH 144/197] test(addons): catch a pinned release that upstream retired An addon may pin `config.archive` so the deploy fetches that release instead of letting the app store choose. The pin is worth exactly as long as the artifact outlives it: when upstream retires the release, every deploy of the role fails at install time with a 404 and nothing in the failure names the declaration as the cause. That is not hypothetical. ONLYOFFICE listed onlyoffice 10.2.0 in the Nextcloud appstore and never published the artifact, which made Nextcloud#4 red in both runs 35329488812 and 35330645276 on the same commit. This check answers the mirror image of that question: does the release we pinned still exist. The split follows the sibling model-pin check. The declaration's shape is a repository fact and is asserted; whether the archive is still served is a network fact, so a timeout or a refused connection reports unverified rather than failing, because a firewalled runner must not turn a good pin red. A 4xx is an answer that contradicts the pin and does fail. Negative control: pointing the pin at the retired v10.2.0 produces web-app-nextcloud.onlyoffice: .../v10.2.0/onlyoffice.tar.gz answered HTTP 404 so the check would have caught today's failure before it reached CI. The unverified path is exercised too: with the sandbox blocking the host, the run warns instead of failing. Residual: this proves reachability, not compatibility. A pin whose max-version sits below the deployed server major passes here and fails loudly later at `app:enable`. Today the single pin is v10.1.2, which the store declares for >=33.0.0 <35.0.0 against a role that deploys 34. Co-Authored-By: Claude Opus 5 (1M context) --- .../test_pinned_addon_archives_available.py | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 tests/external/roles/test_pinned_addon_archives_available.py diff --git a/tests/external/roles/test_pinned_addon_archives_available.py b/tests/external/roles/test_pinned_addon_archives_available.py new file mode 100644 index 0000000000..93d5b8e0d3 --- /dev/null +++ b/tests/external/roles/test_pinned_addon_archives_available.py @@ -0,0 +1,123 @@ +"""Every addon pinned to a release archive still resolves upstream. + +An addon may carry ``config.archive``: the deploy fetches that archive and +unpacks it instead of letting the app store pick the newest release. The pin +exists because a store can advertise a release whose artifact was never +published, and a deploy has no way to fall back from that. + +The pin is worth exactly as long as the artifact outlives it. When upstream +retires the release, every deploy of that role fails at install time with a 404 +and nothing in the failure names the declaration as the cause. The same shape +already cost this repository a red CI job on two runs, in the opposite +direction: the store offered a version whose archive answered 404. + +The declaration's shape is a repository fact and is asserted. Whether the +archive is still served is a network fact, so a timeout or a refused connection +is reported as unverified rather than failing, the way the sibling external +checks do: a firewalled runner must not turn a good pin red. An answer that +contradicts the pin is a defect in the repository and does fail. +""" + +from __future__ import annotations + +import concurrent.futures +import unittest +from pathlib import Path +from urllib.parse import urlsplit + +import requests + +from utils.annotations.message import warning +from utils.cache.yaml import load_yaml_any + +from . import PROJECT_ROOT + +_ADDON_GLOB = "*/meta/addons/*.yml" +_USER_AGENT = "infinito-nexus-addon-pin-check" +_TIMEOUT_SECONDS = 20 +_MAX_WORKERS = 4 + + +def _pinned_archives() -> list[tuple[str, str, str]]: + """Every ``(role, addon, archive)`` an addon pins, in path order.""" + found: list[tuple[str, str, str]] = [] + for addon_file in sorted((PROJECT_ROOT / "roles").glob(_ADDON_GLOB)): + spec = load_yaml_any(str(addon_file), default_if_missing={}) or {} + archive = (spec.get("config") or {}).get("archive") + if not archive: + continue + role = addon_file.parent.parent.parent.name + found.append((role, addon_file.stem, str(archive))) + return found + + +def _relative(role: str, addon: str) -> str: + return str(Path("roles") / role / "meta" / "addons" / f"{addon}.yml") + + +def _reachable(url: str) -> tuple[str, str]: + """Return (kind, detail); a redirect to a release CDN counts as reachable.""" + try: + response = requests.get( + url, + allow_redirects=True, + headers={"User-Agent": _USER_AGENT, "Range": "bytes=0-0"}, + stream=True, + timeout=_TIMEOUT_SECONDS, + ) + try: + status = response.status_code + finally: + response.close() + except requests.Timeout as exc: + return "unverified", f"Timeout: {exc}" + except requests.RequestException as exc: + return "unverified", f"{type(exc).__name__}: {exc}" + + if status < 400: + return "ok", f"HTTP {status}" + return "gone", f"HTTP {status}" + + +class TestPinnedAddonArchivesAvailable(unittest.TestCase): + def test_every_pinned_archive_is_an_absolute_url(self) -> None: + malformed = [ + f"{role}.{addon}: {archive!r} is not an absolute URL" + for role, addon, archive in _pinned_archives() + if not urlsplit(archive).scheme or not urlsplit(archive).netloc + ] + self.assertFalse( + malformed, + f"{len(malformed)} addon pin(s) cannot be fetched. 'config.archive' " + "must be an absolute URL to a release artifact:\n" + + "\n".join(f" {line}" for line in malformed), + ) + + def test_every_pinned_archive_is_still_served(self) -> None: + pins = _pinned_archives() + if not pins: + self.skipTest("no addon pins a release archive") + + with concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_WORKERS) as pool: + answers = list(pool.map(_reachable, [archive for _, _, archive in pins])) + + retired = [] + for (role, addon, archive), (kind, detail) in zip(pins, answers, strict=True): + if kind == "ok": + continue + if kind == "unverified": + warning( + f"{role}.{addon}: {archive} was not checked ({detail})", + title="Addon pin unverified", + file=_relative(role, addon), + ) + continue + retired.append(f"{role}.{addon}: {archive} answered {detail}") + + self.assertFalse( + retired, + f"{len(retired)} pinned addon archive(s) are gone upstream. Every " + "deploy of the role fails at install time until the pin moves to a " + "release that is still published:\n" + + "\n".join(f" {line}" for line in retired), + ) From eef3c3be7167d4987e40c991afc1f983ec80aa83 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 06:31:48 +0200 Subject: [PATCH 145/197] fix(lmstudio): bound the model fetch by size, disk and a fallback source A wedged LM Studio daemon cost CI job 105558224377 three hours: three kills at the global 60-minute `CONTAINER_EXEC_TIMEOUT`, for a 531 MB model that a healthy node pulls in 27 seconds. Nothing in the path was sized against the thing being fetched, and nothing checked whether it could land at all. Size. Each model now declares `bytes`, taken from the value the deploy already printed as "OK (531068224 bytes)", and the pre-pull derives its own timeout per model: max(preload_floor_seconds, bytes / preload_min_bytes_per_second). The floor of 600s protects a small model from a brief stall; the rate of 1 MB/s sits twenty times below the 21.98 MB/s a healthy pull reached and seven thousand times above the 142 B/s the wedged one managed, so it separates the two without guessing. Evaluated against ansible-core's filters: 531 MB and 10 MB both land on the floor, 4 GB gives 4000s. Disk. The seed measures free space with `df -P -B1` before fetching and refuses with a message naming both figures in GiB, the factor and the file to edit. The factor is three because the node cache, the staging copy inside the container and the models volume all exist at once during the import; that is measured, not padding. Fallback. A model may list `mirrors`, extra bases tried after the source. The mirror task is a no-op once the source delivered, because get_url skips a dest whose checksum already matches: measured at 145.24s for the first pass against 0.64s for the second. The list ships empty, because the obvious candidate is not one: hf-mirror.com answers 206 but resolves to us.aws.cdn.hf.co, so it is a front end for HuggingFace and adds no redundancy against a HuggingFace outage. The mechanism is here; an independent copy is the operator's to name. The external check grew with the declaration: `bytes` is compared against the size the API publishes, and every declared mirror must still serve the pinned file, or it is redundancy on paper only. 01_manager_ops.yml's rescue rationale claimed that restarting the container clears a wedged transfer. Job 105558224377 shows the restart returning rc 0 without clearing it, so the claim now says what was measured instead. Residual: the mirror path has never run in a deploy, since no mirror is declared. Its loop expression is verified against ansible-core's subelements over four shapes, including the zero-iteration case when nothing is mirrored. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/16_ai.yml | 1 + roles/svc-ai-lmstudio/meta/services.yml | 2 + .../svc-ai-lmstudio/tasks/01_manager_ops.yml | 2 +- roles/svc-ai-lmstudio/tasks/utils/preload.yml | 5 ++ roles/svc-ai-lmstudio/tasks/utils/seed.yml | 33 ++++++++++ roles/svc-ai-lmstudio/vars/main.yml | 7 ++ .../test_ai_local_models_downloadable.py | 64 +++++++++++++++++-- 7 files changed, 108 insertions(+), 6 deletions(-) diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index e5c6ee6df1..8c47b2dcb1 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -27,6 +27,7 @@ AI_LOCAL_MODELS: source: https://huggingface.co/lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF file: Qwen2.5-0.5B-Instruct-Q8_0.gguf sha256: "1d96e30253a76970864323987e3e77eeaf170312952fd61d18d343c25cfd6e52" + bytes: 531068224 AI_REMOTE_ALIASES: >- {{ ['openai/gpt-4o-mini'] * (lookup('api_enabled', 'openai') | int) diff --git a/roles/svc-ai-lmstudio/meta/services.yml b/roles/svc-ai-lmstudio/meta/services.yml index d27bb1b516..53bab2e9c2 100644 --- a/roles/svc-ai-lmstudio/meta/services.yml +++ b/roles/svc-ai-lmstudio/meta/services.yml @@ -25,6 +25,8 @@ lmstudio: http: 1234 preload_models: "{{ AI_LOCAL_MODELS }}" preload_async_time: 3600 + preload_floor_seconds: 600 + preload_min_bytes_per_second: 1000000 lifecycle: beta # nocheck: playwright-service-flag, dynamic-flag diff --git a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml index 6cffa1413f..ac3c91c989 100644 --- a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml +++ b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml @@ -17,7 +17,7 @@ - name: "โฌ‡๏ธ Preload the LM Studio models (first pass)" ansible.builtin.include_tasks: utils/preload.yml rescue: - - name: "๐Ÿ Re-raise the preload failure unchanged in swarm" # nocheck: rescue-recovery a timed-out transfer leaves the daemon reporting 0.00% for every further pull, and only restarting the container it owns clears that, which the generic capture cannot do; swarm re-raises here instead, because under any_errors_fatal a rescued host keeps running while every other host is already marked failed, and compose ends the block by preloading again so an unrecovered failure still re-raises + - name: "๐Ÿ Re-raise the preload failure unchanged in swarm" # nocheck: rescue-recovery a timed-out transfer leaves the daemon reporting 0.00% for every further pull, and restarting the container it owns is the only lever the role has, though CI job 105558224377 shows a restart returning rc 0 without clearing it; the generic capture cannot even try, so swarm re-raises here instead, because under any_errors_fatal a rescued host keeps running while every other host is already marked failed, and compose ends the block by preloading again so an unrecovered failure still re-raises ansible.builtin.fail: msg: "{{ ansible_failed_result.msg | default('LM Studio preload failed') }}" when: DEPLOYMENT_MODE == 'swarm' diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload.yml b/roles/svc-ai-lmstudio/tasks/utils/preload.yml index a50c35ec9e..6087c739b1 100644 --- a/roles/svc-ai-lmstudio/tasks/utils/preload.yml +++ b/roles/svc-ai-lmstudio/tasks/utils/preload.yml @@ -2,6 +2,11 @@ - name: โฌ‡๏ธ Pre-pull LM Studio models ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} lms get --yes --gguf {{ model.source | quote }} + environment: + CONTAINER_EXEC_TIMEOUT: >- + {{ [ LMSTUDIO_PRELOAD_FLOOR_SECONDS | int, + ((model.bytes | int) / (LMSTUDIO_PRELOAD_MIN_BYTES_PER_SECOND | int)) + | round(0, 'ceil') | int ] | max }} register: lmstudio_pull_result loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" loop_control: diff --git a/roles/svc-ai-lmstudio/tasks/utils/seed.yml b/roles/svc-ai-lmstudio/tasks/utils/seed.yml index 09eb7f45e6..41d2501a3c 100644 --- a/roles/svc-ai-lmstudio/tasks/utils/seed.yml +++ b/roles/svc-ai-lmstudio/tasks/utils/seed.yml @@ -5,6 +5,27 @@ state: directory mode: "0755" +- name: "๐Ÿ—„๏ธ Measure the free space the models have to fit into" + ansible.builtin.command: "df -P -B1 {{ LMSTUDIO_MODEL_CACHE_DIR | quote }}" + register: lmstudio_cache_space + changed_when: false + +- name: "๐Ÿ›‘ Refuse to fetch models the hosting node cannot hold" + vars: + _free: "{{ lmstudio_cache_space.stdout_lines[1].split()[3] | int }}" + _required: "{{ LMSTUDIO_PRELOAD_REQUIRED_BYTES | int }}" + ansible.builtin.fail: + msg: >- + The declared LM Studio models need {{ (_required | int / 1024 / 1024 / 1024) + | round(1) }} GiB on {{ LMSTUDIO_HOST_NODE }} but only {{ (_free | int / 1024 + / 1024 / 1024) | round(1) }} GiB is free. That is + {{ LMSTUDIO_PRELOAD_COPIES_ON_DISK }} times the + {{ (LMSTUDIO_PRELOAD_BYTES | int / 1024 / 1024 / 1024) | round(1) }} GiB the + models declare, because the node cache, the staging copy inside the container + and the models volume all exist at once during the import. Shrink + AI_LOCAL_MODELS in group_vars/all/16_ai.yml or give the node more disk. + when: _free | int < _required | int + - name: "โฌ‡๏ธ Warm the model cache on the node, best effort" get_url_retry: url: "{{ model.source }}/resolve/main/{{ model.file }}" @@ -18,6 +39,18 @@ changed_when: false failed_when: false +- name: "๐Ÿชž Warm the model cache from a mirror when the source stayed silent" + get_url_retry: + url: "{{ item.1 }}/resolve/main/{{ item.0.file }}" + dest: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ item.0.file }}" + checksum: "sha256:{{ item.0.sha256 }}" + mode: "0644" + loop: "{{ LMSTUDIO_PRELOAD_MODELS | subelements('mirrors', skip_missing=True) }}" + loop_control: + label: "{{ item.0.file }} from {{ item.1 }}" + changed_when: false + failed_when: false + - name: "๐Ÿ“‹ Hand the warmed model to LM Studio" ansible.builtin.script: shell/import.sh environment: diff --git a/roles/svc-ai-lmstudio/vars/main.yml b/roles/svc-ai-lmstudio/vars/main.yml index 1dae1fe5e8..3c55763554 100644 --- a/roles/svc-ai-lmstudio/vars/main.yml +++ b/roles/svc-ai-lmstudio/vars/main.yml @@ -6,6 +6,13 @@ LMSTUDIO_CONTAINER: "{{ lookup('config', application_id, 'services.lmstudio.name LMSTUDIO_PORT: "{{ lookup('config', application_id, 'services.lmstudio.ports.internal.http') }}" LMSTUDIO_PRELOAD_MODELS: "{{ lookup('config', application_id, 'services.lmstudio.preload_models') }}" LMSTUDIO_PRELOAD_ASYNC_TIME: "{{ lookup('config', application_id, 'services.lmstudio.preload_async_time') }}" +LMSTUDIO_PRELOAD_FLOOR_SECONDS: "{{ lookup('config', application_id, 'services.lmstudio.preload_floor_seconds') }}" +LMSTUDIO_PRELOAD_MIN_BYTES_PER_SECOND: "{{ lookup('config', application_id, 'services.lmstudio.preload_min_bytes_per_second') }}" +LMSTUDIO_PRELOAD_BYTES: >- + {{ LMSTUDIO_PRELOAD_MODELS | map(attribute='bytes') | map('int') | sum }} +LMSTUDIO_PRELOAD_COPIES_ON_DISK: 3 +LMSTUDIO_PRELOAD_REQUIRED_BYTES: >- + {{ (LMSTUDIO_PRELOAD_BYTES | int) * (LMSTUDIO_PRELOAD_COPIES_ON_DISK | int) }} LMSTUDIO_CONFIG_DIR: "{{ lookup('container', application_id, 'directories.config') }}" LMSTUDIO_MODEL_CACHE_DIR: "{{ [ LMSTUDIO_CONFIG_DIR, 'models' ] | path_join }}" LMSTUDIO_HOME_DIR: "{{ lookup('volume', application_id, 'lmstudio_data').target }}" diff --git a/tests/external/roles/test_ai_local_models_downloadable.py b/tests/external/roles/test_ai_local_models_downloadable.py index cd6785ed74..cc44488f16 100644 --- a/tests/external/roles/test_ai_local_models_downloadable.py +++ b/tests/external/roles/test_ai_local_models_downloadable.py @@ -52,11 +52,22 @@ def _download_url(entry: dict) -> str: return f"{entry['source'].rstrip('/')}/resolve/main/{entry['file']}" +def _mirror_urls(entry: dict) -> list[str]: + """Every extra base a deploy falls back to, in declaration order.""" + mirrors = entry.get("mirrors") or [] + return [ + f"{str(base).rstrip('/')}/resolve/main/{entry['file']}" + for base in mirrors + if isinstance(base, str) + ] + + def _upstream_digests(repo_path: str) -> tuple[str, object]: """Return (kind, payload) with kind one of ok, warn, unverified. - On ``ok`` the payload maps each published file name to its sha256, which is - what the pin is compared against without transferring the file itself. + On ``ok`` the payload maps each published file name to its sha256 and byte + count, which is what the pin is compared against without transferring the + file itself. """ url = f"{_HUGGINGFACE_PREFIX}api/models/{repo_path}" try: @@ -84,8 +95,9 @@ def _upstream_digests(repo_path: str) -> tuple[str, object]: continue name = sibling.get("rfilename") digest = (sibling.get("lfs") or {}).get("sha256") + size = (sibling.get("lfs") or {}).get("size") or sibling.get("size") if isinstance(name, str) and isinstance(digest, str): - digests[name] = digest + digests[name] = {"sha256": digest, "size": size} return "ok", digests @@ -130,6 +142,9 @@ class TestAiLocalModelsDownloadable(unittest.TestCase): unpinned.append(f"{alias}: 'file' is missing") if not isinstance(digest, str) or not _SHA256.match(digest): unpinned.append(f"{alias}: 'sha256' is not 64 hex characters") + size = entry.get("bytes") + if not isinstance(size, int) or size <= 0: + unpinned.append(f"{alias}: 'bytes' is not a positive integer") self.assertFalse( unpinned, @@ -183,11 +198,18 @@ class TestAiLocalModelsDownloadable(unittest.TestCase): mismatched.append( f"{alias}: {repo_path} no longer publishes {entry['file']}" ) - elif published != entry["sha256"]: + continue + if published["sha256"] != entry["sha256"]: mismatched.append( - f"{alias}: {entry['file']} is {published} upstream, " + f"{alias}: {entry['file']} is {published['sha256']} upstream, " f"pinned as {entry['sha256']}" ) + pinned_size = entry.get("bytes") + if published["size"] is not None and published["size"] != pinned_size: + mismatched.append( + f"{alias}: {entry['file']} is {published['size']} bytes " + f"upstream, pinned as {pinned_size}" + ) self.assertFalse( mismatched, @@ -197,6 +219,38 @@ class TestAiLocalModelsDownloadable(unittest.TestCase): + "\n".join(f" {line}" for line in mismatched), ) + def test_every_declared_mirror_still_serves_the_file(self) -> None: + targets = [ + (str(entry.get("alias") or ""), url) + for entry in _models() + for url in _mirror_urls(entry) + ] + if not targets: + self.skipTest(f"{_AI_VARS_FILE} declares no model mirror") + + with concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_WORKERS) as pool: + answers = list(pool.map(_reachable, [url for _, url in targets])) + + broken = [] + for (alias, url), (kind, detail) in zip(targets, answers, strict=True): + if kind == "ok": + continue + if kind == "unverified": + warning( + f"{alias}: mirror {url} was not checked ({detail})", + title="Model mirror unverified", + file=_AI_VARS_FILE, + ) + continue + broken.append(f"{alias}: mirror {url} answered {detail}") + + self.assertFalse( + broken, + f"{len(broken)} declared mirror(s) no longer serve the pinned file, so " + "they add no redundancy. Drop the entry or point it at a copy that " + "answers:\n" + "\n".join(f" {line}" for line in broken), + ) + if __name__ == "__main__": # pragma: no cover unittest.main() From 93990bbf0762d7bc626ff1b08e817d35ba4d2843 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 06:32:09 +0200 Subject: [PATCH 146/197] docs(claude): say that the mandated tee pipe throws the exit status away MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The logging rule requires streaming through `โ€ฆ 2>&1 | tee /tmp/.log`, and that pipe reports tee's status, not make's. An agent following the rule as written reads exit 0 from a run that ended in `Error 2`. This happened twice on 2026-09-19. Both times `make test` printed `FAILED TARGETS: test-lint` and the invocation returned 0; only reading the table caught it, once on a genuine addon-schema violation that would otherwise have been committed. The rule now names the trade it makes, so the table is the verdict and the exit code is not. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CLAUDE.md b/CLAUDE.md index 7c96f38bcc..8487cc9e7c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,6 +41,7 @@ Anything else โ€” restating code, section banners, "Note that โ€ฆ", step narrati - For ANY non-trivial command (test runs, deploys, long pipelines), you MUST stream the FULL output to a file under `/tmp/` via `โ€ฆ 2>&1 | tee /tmp/.log` and grep / inspect that file repeatedly instead of re-running the command. Reason: re-running `make test` to "find the failure I just lost" costs 2 minutes per cycle; grepping the saved log costs milliseconds. - Default the filename to a meaningful slug + monotonically increasing index (`/tmp/make-test--.log`, `/tmp/act--.log`) so you can compare runs. - Tell the operator the exact `tail -f /tmp/.log` command for background runs, per the existing rule. +- That pipe **destroys the exit status**: `make test | tee` reports `tee`'s success even when make ends with `Error 2`. You MUST judge such a run by its `๐Ÿ“Š per-target wall-clock` table and its `FAILED TARGETS:` line, never by the exit code. ## CI evidence ๐Ÿ“Š From 3896e88329e9089e2a6ebd531365959679fb0a48 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 06:41:25 +0200 Subject: [PATCH 147/197] test(filters): allow ansible-core's subelements in the filter allowlist The mirror loop added in eef3c3be71 iterates with `LMSTUDIO_PRELOAD_MODELS | subelements('mirrors', skip_missing=True)`. That filter ships with ansible-core rather than with this repository's `filter_plugins/`, and the allowlist that separates "provided by the runtime" from "must be defined here" did not carry it, so test-integration went red on a filter that is perfectly valid: These filters are used in templates/YAML but have no local definition (and are not in BUILTIN_FILTERS): - subelements Measured against the interpreter the suite runs on, `FilterModule().filters()` from `ansible.plugins.filter.core` contains `subelements`, so the entry states a fact rather than a suppression. The set has exactly one reader, the subtraction in the same file, and the entry removes that one name from the candidates and nothing else. This is the same one-line extension as 9118b47654 (regex_findall) and fa59b29b2a (urlsplit). Residual: the check can no longer report this name, so a future local filter called `subelements` would shadow the builtin unnoticed. Co-Authored-By: Claude Opus 5 (1M context) --- tests/integration/ansible/filters/test_defined.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/ansible/filters/test_defined.py b/tests/integration/ansible/filters/test_defined.py index 3b531346e4..cf8cd59d76 100644 --- a/tests/integration/ansible/filters/test_defined.py +++ b/tests/integration/ansible/filters/test_defined.py @@ -51,6 +51,7 @@ BUILTIN_FILTERS: set[str] = { "sort", "string", "striptags", + "subelements", "sum", "title", "trim", From 75744c84be6b5c27ab50b4b96304d99b0b737576 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 07:41:01 +0200 Subject: [PATCH 148/197] fix(make): stop a repeated lint target from hiding its own failure The lint fan-out listed lint-php twice, once before and once after lint-playwright. Two commits added it independently: 82218087c1 for the PHP and Ruby syntax checks, 6fee7ca136 for the per-language unit suites, and the second did not see the first because lint-playwright had since been inserted between them. The wasted slot was the harmless half. parallel.sh keys target_start, target_dur, target_rc and target_log by target NAME, so a repeat overwrites the first run's log path and exit code, and the final pass reads each name once. Two consequences: drain_one prints the second run's log twice and the first run's output never appears, and a failing first invocation is reported green as soon as the second one passes. A duplicate therefore turns a target into a blind spot rather than into a slow one. The gate showed the symptom plainly, two lint-php rows with an identical 0m 03s, which reads as a rendering quirk rather than as one target run twice. Both halves are fixed: the list loses the repeat, and parallel.sh refuses a duplicate before it loads the environment or starts a single make, naming the offender and why a repeat cannot be tolerated. The two callers are Makefile:440 and Makefile:822; the test list has no duplicates, so the guard is inert there. Negative control: `parallel.sh lint-php lint-python lint-php` exits 2 with "duplicate target(s): lint-php" and launches nothing. The full quality-high run afterwards dispatches 15 lint targets with lint-php once and stays green. Residual: the guard covers scripts/make/parallel.sh only. scripts/tests/dns/ parallel.sh is a separate script with its own argument handling and was not examined. Co-Authored-By: Claude Opus 5 (1M context) --- Makefile | 1 - scripts/make/parallel.sh | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b743565ca4..605131bfe7 100644 --- a/Makefile +++ b/Makefile @@ -448,7 +448,6 @@ lint: install-lint lint-packages \ lint-php \ lint-playwright \ - lint-php \ lint-python \ lint-ruby \ lint-shellcheck \ diff --git a/scripts/make/parallel.sh b/scripts/make/parallel.sh index 0140e2db07..83a5678f41 100755 --- a/scripts/make/parallel.sh +++ b/scripts/make/parallel.sh @@ -14,6 +14,12 @@ if [[ $# -eq 0 ]]; then exit 2 fi +duplicates="$(printf '%s\n' "$@" | sort | uniq -d)" +if [[ -n "${duplicates}" ]]; then + echo "scripts/make/parallel.sh: duplicate target(s): ${duplicates//$'\n'/ }; each table here is keyed by target name, so a repeat overwrites the first run's exit code and log" >&2 # nocheck: self-path-reference + exit 2 +fi + # shellcheck source=scripts/meta/env/load.sh source scripts/meta/env/load.sh From 9b436a3162d79588d1ce15bd38d752ee7c5344af Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 07:44:02 +0200 Subject: [PATCH 149/197] fix(cache): key the project-tree walk on the root it walked _all_project_files carried @lru_cache(maxsize=1) and read PROJECT_ROOT from the module global inside its body, so the root was not part of the cache key. The first call decided the answer for the rest of the process: point PROJECT_ROOT at a fixture afterwards and the walk still reports the real repository, or, in the other order, a sweep after the fixture is torn down enumerates a directory that no longer exists. Both failure modes are silent. iter_project_files, iter_project_files_with_content and iter_non_ignored_files all sit on this one walk, 202 files in the tree use them, and a lint whose corpus collapses to zero paths reports zero offenders and passes. Nothing raises. The root is now a parameter and therefore the key, with room for four of them. _reset() still clears it through cache_clear(), and the only other caller is iter_project_files, which passes str(PROJECT_ROOT) explicitly. Live impact today is none, and that is worth stating: exactly one test patches PROJECT_ROOT in this module, and it calls _reset() in both setUp and addCleanup, so the defect was fully masked. Every other patch site in the repo targets PROJECT_ROOT in a different module. This is about the next test that forgets, not about a failure anyone has seen. A regression test covers it rather than a comment. Negative control: with the key removed the new test fails on the second root with Items in the first set but not the second: '/tmp/.../first.txt' and passes once the key is back; the file is byte-identical to its pre-probe state afterwards. The lint docstring that quoted the old cache shape is deleted rather than updated, because a cross-module implementation reference goes stale the moment that module changes, which is exactly what happened here. Same class as c4556fe72d, which keyed the applications SPOT by the groups its payload reads. That one was a live defect in utils/cache/base.py; this is the latent twin one module over. Residual: maxsize=4 is a choice, not a measurement. More than four roots alive at once evicts the oldest, which re-walks rather than answers wrongly. Co-Authored-By: Claude Opus 5 (1M context) --- .../test_role_and_group_vars_used.py | 3 +- tests/unit/python/utils/cache/test_files.py | 28 +++++++++++++++++++ utils/cache/files.py | 9 +++--- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/lint/ansible/variables/test_role_and_group_vars_used.py b/tests/lint/ansible/variables/test_role_and_group_vars_used.py index d663e8e1e1..89fa90a7f9 100644 --- a/tests/lint/ansible/variables/test_role_and_group_vars_used.py +++ b/tests/lint/ansible/variables/test_role_and_group_vars_used.py @@ -194,8 +194,7 @@ def _build_usage_indices(repo_root: Path) -> tuple[set[str], set[str], set[str]] Routes every read through the project's cached helpers: * ``iter_project_files_with_content`` walks the path list once - per process (``lru_cache(maxsize=1)`` on `_all_project_files`) - and reads each file via the cached ``read_text``; + per process and reads each file via the cached ``read_text``; * ``load_yaml_any`` parses each YAML file once per ``(path, mtime, size)`` signature and reuses the parsed structure on every later call in the same process โ€” no diff --git a/tests/unit/python/utils/cache/test_files.py b/tests/unit/python/utils/cache/test_files.py index b614ed41db..14b554017e 100644 --- a/tests/unit/python/utils/cache/test_files.py +++ b/tests/unit/python/utils/cache/test_files.py @@ -164,5 +164,33 @@ class TestReset(_ProjectRootFixture, unittest.TestCase): self.assertEqual(read_text(str(self.root / "x.txt")), "v1-changed") +class TestWalkIsKeyedOnTheRoot(_ProjectRootFixture, unittest.TestCase): + def test_a_second_root_is_walked_rather_than_answered_from_the_first( + self, + ) -> None: + _touch(self.root / "first.txt", "a") + self.assertEqual( + {str(self.root / "first.txt")}, + set(iter_project_files()), + ) + + with TemporaryDirectory() as other: + other_root = Path(other) + _touch(other_root / "second.txt", "b") + with patch.object(files_module, "PROJECT_ROOT", other_root): + self.assertEqual( + {str(other_root / "second.txt")}, + set(iter_project_files()), + "the walk answered for the previous root, so a test that " + "points PROJECT_ROOT at a fixture silently sweeps the real " + "repository instead", + ) + + self.assertEqual( + {str(self.root / "first.txt")}, + set(iter_project_files()), + ) + + if __name__ == "__main__": # pragma: no cover unittest.main() diff --git a/utils/cache/files.py b/utils/cache/files.py index 42249b8a39..6efd2d95c4 100644 --- a/utils/cache/files.py +++ b/utils/cache/files.py @@ -54,15 +54,14 @@ _DEFAULT_SKIP_DIRS: frozenset[str] = frozenset( ) -@lru_cache(maxsize=1) -def _all_project_files() -> tuple[str, ...]: - """One-shot filesystem walk from PROJECT_ROOT. +@lru_cache(maxsize=4) +def _all_project_files(root_str: str) -> tuple[str, ...]: + """One-shot filesystem walk from *root_str*. Returns absolute paths as a tuple (immutable โ†’ safe to cache). Pruned at walk-time by `_DEFAULT_SKIP_DIRS`, since those dirs are never relevant for any project-tree test and descending into them is pure waste. """ - root_str = str(PROJECT_ROOT) paths: list[str] = [] for dirpath, dirnames, filenames in os.walk(root_str, topdown=True): dirnames[:] = [d for d in dirnames if d not in _DEFAULT_SKIP_DIRS] @@ -87,7 +86,7 @@ def iter_project_files( ``exclude_dirs=("docs",)`` skips any file whose path contains a ``docs`` segment. """ - all_files = _all_project_files() + all_files = _all_project_files(str(PROJECT_ROOT)) tests_prefix = str(Path(str(PROJECT_ROOT)) / "tests") + os.sep ext_lower = tuple(e.lower() for e in extensions) if extensions else None exclude_dirs_set = set(exclude_dirs) if exclude_dirs else None From 720a1a96effcc91225f03c3bb7d5ee525a53af93 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 07:46:18 +0200 Subject: [PATCH 150/197] perf(test): pay a class fixture once instead of once per xdist worker run.sh drives every suite with -n auto and no scope, so xdist hands out single test METHODS. Each worker is its own process, so setUpClass runs once per worker the class happens to be scattered over, and an expensive fixture is paid as many times as the scheduler split its methods. The integration report shows the shape without interpretation: TestMcpEnvRender's nine methods came back as six at 277-287s and three at 0.003s. The fast three landed on workers that had already paid. That is 1776 of the suite's 2282 CPU-seconds spent computing one identical result six times, twenty Ansible templar renders of the MCP providers each time. --dist loadscope groups by class, so a class occupies one worker and its fixture runs once. 1a908a6e5b introduced -n auto and hoisted fixtures in the same commit, so the hoisting was undercut from the start: work moved from per-method to per-class, and the scheduler it arrived with promptly charged it per worker instead. This finishes that commit rather than correcting it. Measured against the same tree, four suites, before and after: integration 296.83s -> 219.32s lint 105.64s -> 84.27s unit 65.74s -> 53.02s external 75.69s -> 71.99s Phase 1's critical path drops from 4m57s to 3m40s. That unit and lint move too says the pattern was not unique to the MCP class. Coverage is unchanged, which is the point of the change rather than a side note. All four suites report identical counts: unit 5857 passed / 9 skipped / 2301 subtests, external 19 / 2, lint 521 / 1 / 374, integration 328 / 2 / 1562. A second effect is worth naming because it cuts the right way. Several sweeps keep their non-empty guard in a separate method, e.g. test_fragments.py:139 proving the corpus is not empty before the sweep asserts over it. Under --dist load those two ran in different processes with separate caches, so the guard proved nothing about the process the sweep ran in. loadscope puts them back together. Residual: loadscope trades balance for locality, and here one class sets the floor. TestMcpEnvRender holds a worker for ~250s on its own, so the suite cannot go below that until the ~12s per render is addressed. That cost is the render guard in utils/cache/applications.py:415 returning the merged map without caching it, which is production config resolution and deliberately left alone. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/tests/code/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/code/run.sh b/scripts/tests/code/run.sh index bcec6268d4..b4c74106c9 100755 --- a/scripts/tests/code/run.sh +++ b/scripts/tests/code/run.sh @@ -35,7 +35,7 @@ if "${PYTHON}" -c 'import pytest, xdist' >/dev/null 2>&1; then # the exact set `unittest discover` runs. PYTHONDONTWRITEBYTECODE keeps # namespace probing from dropping __pycache__ into roles/. mkdir -p build/test-reports - PYTHONDONTWRITEBYTECODE=1 "${PYTHON}" -m pytest "${_suite_dir}" -q -n auto -rP -p no:cacheprovider \ + PYTHONDONTWRITEBYTECODE=1 "${PYTHON}" -m pytest "${_suite_dir}" -q -n auto --dist loadscope -rP -p no:cacheprovider \ --import-mode=importlib \ --junitxml="${_junit_report}" \ -o consider_namespace_packages=true \ From c6e8990e25704a61b1f0fb67c510caaff7bd0e37 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 09:12:21 +0200 Subject: [PATCH 151/197] fix(lmstudio): keep the daemon's auth key off the shared volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Job 105845259891 (svc-ai-lmstudio#0@swarm, debian/btrfs) died in the model pre-pull, two seconds in, not on a timeout: Error: Failed to authenticate: Provided authPacket lms-cli is not allowed. The daemon was up and rejecting deliberately. Its own log carries the server half of the same line, through SystemExternalAPIProvider. Reproduced locally in compose, without swarm and without NFS, which is what pins the cause. Two daemons sharing one /root/.lmstudio: daemon 1 writes .internal/lms-key-2 md5 c82691a9b6cb54c0eb233d2842f9f358 daemon 2 rewrites it md5 e407c5e17d7392b88080e8ed216f641d lms against daemon 1 Provided authPacket lms-cli is not allowed The key is the runtime half of the CLI's passkey and it lives in .internal, not in credentials/, which is empty. /app/daemon-run.sh starts `lms server start` in the entrypoint, so the CLI authenticates at container start and the loser of the race is whichever daemon started first. Swarm only supplied the second daemon: the role pins no replicas, so compose_replicas defaults to one task per host, and /root/.lmstudio carries no `nfs: false`, so the swarm rewrite puts all three on one NFS export. The diagnostics show three task slots and five .ready- markers in one directory. .internal now has its own node-local volume. The models stay shared, which is the point: the blobs are what the nodes may share, the daemon's identity is not. Deobfuscating the daemon bundle confirms the split lines up with the vendor's own layout: modelIndexCache, ggufMetadataCache and modelData resolve inside .internal, defaultModelsFolder and hubModelsFolder resolve outside it, and the folder watcher explicitly skips .internal when looking for models. A missing index is a supported cold start: the accessor falls back to emptyModelIndex with neverError, and startup re-runs the full directory walk. `name:` is pinned because backup_volume.py emits `entry.get("name") or semantic_name`, while the volume materialises as _. Without the pin the exclusion would name lmstudio_internal, the volume would be lmstudio_lmstudio_internal as the run's artifact shows, and `backup: false` would silently miss. Verified against a real three-node swarm with NFS, the same shape that was red: the pre-pull ran `ok: [mgr-01 -> wrk-02] => (item=qwen2.5:0.5b)`, the service reports 3/3, `lms ls` inside a container lists the model with no auth error, and the job ends ๐Ÿ Job succeeded with no fatal and no failed host. Shape follows web-app-matomo, whose `tmp` is an `nfs: false` child volume nested in an NFS-backed parent with the same nocheck; 32 such nested pairs exist across 9 roles. Residual, and it is not small: multi-replica operation stays unsafe for other reasons this does not touch. extensions/plugins is rm -rf'd and re-extracted at every daemon start, settings.json is written in place and resets itself to defaults on a torn read, bin/lms is a 102 MB copy rewritten while peers execute it, and model downloads land on their final shared path with no temp file. All of that was equally true before this commit; a shared .internal merely added the auth collision on top, and the index is opened noWatch so peers would have overwritten each other's catalogue anyway. Making that safe means either one replica or a fully node-local lmstudio_data with a per-node preload. Second residual, affecting an earlier commit rather than this one: on a node whose index does not know a model whose file is already in the shared models folder, `lms get` sits at 0.00% until it is killed. That is the signature df2fc5084a attributed to a wedged daemon and eef3c3be71 bounded with a dynamic timeout. The seed imports the file and the pre-pull then asks for it again, so the wedge may be this collision rather than the daemon. Measured once, on one node; not yet a diagnosis. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-lmstudio/meta/volumes.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/roles/svc-ai-lmstudio/meta/volumes.yml b/roles/svc-ai-lmstudio/meta/volumes.yml index 51f7e8af72..ae7259ecdf 100644 --- a/roles/svc-ai-lmstudio/meta/volumes.yml +++ b/roles/svc-ai-lmstudio/meta/volumes.yml @@ -5,3 +5,13 @@ lmstudio_data: mounts: - service: lmstudio target: /root/.lmstudio + +lmstudio_internal: + type: volume + name: lmstudio_internal + # nocheck: nfs-false-data-loss Reason: regenerable per-daemon cache, rebuilt in full on first start against an empty volume; shared, a second daemon overwrites the auth key lms-key-2 and the first one's CLI is rejected + nfs: false + backup: false + mounts: + - service: lmstudio + target: /root/.lmstudio/.internal From 6b78ca75788c7f9d78c129399a9ff12955b394d7 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 10:06:20 +0200 Subject: [PATCH 152/197] fix(lmstudio): tolerate an already-present model by what the CLI actually says The preload treats "the model is already there" as success in three places, and all three were dead. They match on the literal `already downloaded`, which the shipped CLI never emits: `strings` over /app/.bundle/lms in the deployed image returns zero hits. What it prints is You already have this model. You can load it with: One of the three was doubly dead. The `until:` reads `stdout`, but the CLI builds its logger as const console = new console$1.Console({ stdout: process.stderr, stderr: process.stderr, so every level including info goes to stderr and that clause could never fire whatever string it carried. The other two are fine on the stream: `failed_when` concatenates stdout and stderr, and the reap path goes through async_failures._said(), which joins msg, stdout and stderr. Effect until now: an already-present model was tolerated only when `lms get` happened to exit 0. On a non-zero exit the task failed with the model sitting on disk, and none of the three guards could catch it. This matters more from here on, not less. The role is moving towards seeding the model before the daemons start, which makes "already present" the normal case rather than the exception. Measured, not inferred: both strings and the logger construction were read out of image 48abf3b7e061, the one the failing CI job ran. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-lmstudio/tasks/utils/preload.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload.yml b/roles/svc-ai-lmstudio/tasks/utils/preload.yml index 6087c739b1..162812fb1c 100644 --- a/roles/svc-ai-lmstudio/tasks/utils/preload.yml +++ b/roles/svc-ai-lmstudio/tasks/utils/preload.yml @@ -19,11 +19,11 @@ delay: 15 until: > (lmstudio_pull_result.rc | default(0)) == 0 or - ('already downloaded' in (lmstudio_pull_result.stdout | default(''))) + ('You already have this model' in (lmstudio_pull_result.stderr | default(''))) changed_when: false failed_when: >- (lmstudio_pull_result.failed | default(false)) - and 'already downloaded' not in ((lmstudio_pull_result.stdout | default('')) ~ (lmstudio_pull_result.stderr | default(''))) + and 'You already have this model' not in ((lmstudio_pull_result.stdout | default('')) ~ (lmstudio_pull_result.stderr | default(''))) delegate_to: "{{ LMSTUDIO_HOST_NODE }}" - name: "โณ Reap the LM Studio model pre-pulls" @@ -35,5 +35,5 @@ async_reap_jobs: "{{ lmstudio_pull_result.results | default([]) }}" async_reap_label: "LM Studio model pre-pull" async_reap_tolerate: - - "already downloaded" + - "You already have this model" ASYNC_TIME: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME }}" From 0b7a8a95dc45caa3d7e73a19b849277c16dcffb4 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 11:50:29 +0200 Subject: [PATCH 153/197] feat(lmstudio): seed the model store once, before the daemons index it c6e8990e25 kept the daemon's auth key off the shared volume and made the red job green, but it left multi-replica operation as broken as it found it: three daemons still shared one /root/.lmstudio, where LM Studio rewrites extensions/plugins with rm -rf at every start, writes settings.json in place and resets it to defaults on a torn read, and re-copies a 102 MB bin/lms while peers execute it. Splitting the tree by access pattern fixes that. /root/.lmstudio is node-local, because it is per-daemon state the daemon rewrites in place. models/ stays shared, because a blob is written once and read forever. That leaves one question, and it is temporal rather than mechanical: the daemon discovers models by an unconditional scan at startup and by an inotify watcher, and inotify never sees writes made by another NFS client. So a model seeded after a daemon started is invisible to it, whichever tool wrote the file. The previous design answered that with `container service update --force`, which restarts the whole inference backend svc-ai-litellm points at, on every deploy. The seed now runs before the stack does. 00_core.yml places the blob on the NFS export ahead of the compose include that deploys the stack, so every daemon finds a complete tree at its own startup scan and nothing has to be restarted. Measured on a three-node swarm with NFS: Fetch the models into the shared store 136.14s one 531 MB download Pre-pull on mgr-01 0.69s Pre-pull on wrk-01 0.50s Pre-pull on wrk-02 0.49s Half a second is not a download. All three daemons recognised the pre-placed model, so the cluster now stores and transfers it once instead of three times. The nested mount across two devices raises no EXDEV, and the per-node disk guard correctly skipped itself four times because the pre-seed covers that check. Vendor coupling is gone from the role. `repo` and `url` are declared fields rather than pieces cut out of a HuggingFace URL with a regex; a model on any host needs only those two. `lms get` still knows only HuggingFace, so `source` stays for it, and a non-HF model simply leaves it unset: the blob is seeded and the startup scan picks it up. The external check follows, comparing against the HuggingFace API only for entries whose url actually points there. The guard arithmetic was wrong and is now derived rather than guessed. Peak usage is the missing bytes plus the largest single model, not twice the declared total, so the read-back moved ahead of the check and subtracts what the store already holds. Before, the requirement grew after every successful seed. Staging moved beside the tree instead of inside it, so an aborted transfer cannot leave orphans where the indexer scans. A deployment test ships with it and states its findings rather than its exit code: it fails on `authPacket` by naming the shared key, on a missing or mismatching blob by naming path and digest, and on a model that is on disk but absent from /v1/models by saying the startup scan did not happen. Gone with the restart: import.sh and its container-cp suppression, seed.yml, rescan.yml, and the disk-copy factor of three that only existed because the import staged through the container. Residual: compose is untested here. There the pre-seed is off, the node-local guard is the one that runs, and `lms get` is the only fetch, so a non-HF model is not loadable in that mode. Seeding node-local volumes is the follow-up that closes it. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/16_ai.yml | 2 + roles/svc-ai-lmstudio/files/shell/import.sh | 34 ----- roles/svc-ai-lmstudio/files/test/.nocheck | 1 + roles/svc-ai-lmstudio/files/test/test.sh | 100 +++++++++++++++ roles/svc-ai-lmstudio/meta/tests.yml | 4 + roles/svc-ai-lmstudio/meta/volumes.yml | 14 +-- roles/svc-ai-lmstudio/tasks/00_core.yml | 8 ++ .../svc-ai-lmstudio/tasks/01_manager_ops.yml | 32 ++--- roles/svc-ai-lmstudio/tasks/utils/cid.yml | 22 ++++ roles/svc-ai-lmstudio/tasks/utils/nodes.yml | 29 +++++ .../tasks/utils/preload/node.yml | 42 +++++++ .../utils/{preload.yml => preload/pull.yml} | 0 .../tasks/utils/{ => preload}/recover.yml | 2 +- roles/svc-ai-lmstudio/tasks/utils/preseed.yml | 118 ++++++++++++++++++ roles/svc-ai-lmstudio/tasks/utils/seed.yml | 65 ---------- roles/svc-ai-lmstudio/templates/test.env.j2 | 5 + roles/svc-ai-lmstudio/vars/main.yml | 20 +-- .../test_ai_local_models_downloadable.py | 40 +++--- 18 files changed, 380 insertions(+), 158 deletions(-) delete mode 100644 roles/svc-ai-lmstudio/files/shell/import.sh create mode 100644 roles/svc-ai-lmstudio/files/test/.nocheck create mode 100644 roles/svc-ai-lmstudio/files/test/test.sh create mode 100644 roles/svc-ai-lmstudio/meta/tests.yml create mode 100644 roles/svc-ai-lmstudio/tasks/utils/cid.yml create mode 100644 roles/svc-ai-lmstudio/tasks/utils/nodes.yml create mode 100644 roles/svc-ai-lmstudio/tasks/utils/preload/node.yml rename roles/svc-ai-lmstudio/tasks/utils/{preload.yml => preload/pull.yml} (100%) rename roles/svc-ai-lmstudio/tasks/utils/{ => preload}/recover.yml (95%) create mode 100644 roles/svc-ai-lmstudio/tasks/utils/preseed.yml delete mode 100644 roles/svc-ai-lmstudio/tasks/utils/seed.yml create mode 100644 roles/svc-ai-lmstudio/templates/test.env.j2 diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index 8c47b2dcb1..c79faa45c5 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -24,6 +24,8 @@ LITELLM_LMSTUDIO_BACKEND: >- AI_LOCAL_MODELS: - alias: qwen2.5:0.5b name: qwen2.5-0.5b-instruct + repo: lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF + url: https://huggingface.co/lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/Qwen2.5-0.5B-Instruct-Q8_0.gguf source: https://huggingface.co/lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF file: Qwen2.5-0.5B-Instruct-Q8_0.gguf sha256: "1d96e30253a76970864323987e3e77eeaf170312952fd61d18d343c25cfd6e52" diff --git a/roles/svc-ai-lmstudio/files/shell/import.sh b/roles/svc-ai-lmstudio/files/shell/import.sh deleted file mode 100644 index 5e9e848d40..0000000000 --- a/roles/svc-ai-lmstudio/files/shell/import.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Hand a pre-fetched GGUF to LM Studio, idempotent. -# Runs ON THE HOSTING NODE (delegated); targets the local container via -# `container exec` / `container cp`. -# -# Required env, supplied by the calling Ansible task: -# LMSTUDIO_LOCAL_CID resolved container id, local to this node -# MODEL_FILE node-side path to the fetched .gguf -# MODEL_USER_REPO HuggingFace "user/repo" the file belongs to -# MODELS_DIR models folder inside the container, under the -# volume target declared in meta/volumes.yml -set -euo pipefail - -ADDRESS="${LMSTUDIO_LOCAL_CID:?LMSTUDIO_LOCAL_CID env var (container id) required}" -BASE="$(basename "${MODEL_FILE}")" -TARGET="${MODELS_DIR}/${MODEL_USER_REPO}/${BASE}" - -if container exec "$ADDRESS" test -f "${TARGET}"; then - echo "PRESENT:${BASE}" - exit 0 -fi - -if [ ! -f "${MODEL_FILE}" ]; then - echo "ABSENT:${BASE}" - exit 0 -fi - -# nocheck: container-cp - MODEL_FILE is fetched on the node this script runs on -container cp "${MODEL_FILE}" "${ADDRESS}:/tmp/${BASE}" - -printf 'y\n' | container exec -i "$ADDRESS" lms import "/tmp/${BASE}" --yes --copy --user-repo "${MODEL_USER_REPO}" -container exec "$ADDRESS" rm -f "/tmp/${BASE}" -echo "IMPORTED:${BASE}" diff --git a/roles/svc-ai-lmstudio/files/test/.nocheck b/roles/svc-ai-lmstudio/files/test/.nocheck new file mode 100644 index 0000000000..a0fdb0bf8c --- /dev/null +++ b/roles/svc-ai-lmstudio/files/test/.nocheck @@ -0,0 +1 @@ +# nocheck: role-files-layout - Role test harness with test.sh as its entry point; the CLI runner stages roles//files/test/ by path. diff --git a/roles/svc-ai-lmstudio/files/test/test.sh b/roles/svc-ai-lmstudio/files/test/test.sh new file mode 100644 index 0000000000..fc83bfc748 --- /dev/null +++ b/roles/svc-ai-lmstudio/files/test/test.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# +# Env (rendered into test.env from templates/test.env.j2): +# LMSTUDIO_CONTAINER resolved container id on this node +# LMSTUDIO_PORT the daemon's HTTP port +# LMSTUDIO_EXPECTED_MODELS JSON array of model names the daemon must serve +# LMSTUDIO_EXPECTED_BLOBS JSON array of / paths under models/ +# LMSTUDIO_EXPECTED_DIGESTS JSON array of sha256, aligned with the blobs +# RETRIES attempts (default 30) +# SLEEP_SECONDS wait between (default 10) + +set -uo pipefail + +RETRIES="${RETRIES:-30}" +SLEEP_SECONDS="${SLEEP_SECONDS:-10}" +MODELS_DIR=/root/.lmstudio/models + +[ -n "${LMSTUDIO_CONTAINER}" ] || { + echo "[FATAL] LMSTUDIO_CONTAINER unset; the harness did not resolve a container" >&2 + exit 2 +} + +in_container() { + container exec "${LMSTUDIO_CONTAINER}" "$@" +} + +json_items() { + python3 -c 'import json,sys; [print(x) for x in json.loads(sys.argv[1])]' "$1" +} + +serves_model() { + in_container curl -fsS --connect-timeout 5 --max-time 30 "http://127.0.0.1:${LMSTUDIO_PORT}/v1/models" 2>/dev/null | + python3 -c 'import json,sys; d=json.load(sys.stdin); print("\n".join(m.get("id","") for m in d.get("data",[])))' 2>/dev/null | + grep -qxF "$1" +} + +failed=0 + +echo "=== 1. the CLI authenticates against its own daemon ===" +attempt=1 +auth_out="" +while [ "${attempt}" -le "${RETRIES}" ]; do + if auth_out="$(in_container /app/.bundle/lms ls 2>&1)"; then + echo "[OK] lms ls authenticated" + break + fi + if printf '%s' "${auth_out}" | grep -q 'authPacket'; then + echo "[FAIL] the daemon rejected its own CLI: $(printf '%s' "${auth_out}" | grep -m1 authPacket)" >&2 + echo " a second daemon shares /root/.lmstudio and rewrote .internal/lms-key-2" >&2 + exit 1 + fi + attempt=$((attempt + 1)) + [ "${attempt}" -le "${RETRIES}" ] && sleep "${SLEEP_SECONDS}" +done +if [ "${attempt}" -gt "${RETRIES}" ]; then + echo "[FAIL] lms ls never succeeded after ${RETRIES} attempts: ${auth_out}" >&2 + exit 1 +fi + +echo "=== 2. every declared blob is in place with its declared digest ===" +mapfile -t blobs < <(json_items "${LMSTUDIO_EXPECTED_BLOBS}") +mapfile -t digests < <(json_items "${LMSTUDIO_EXPECTED_DIGESTS}") +for i in "${!blobs[@]}"; do + path="${MODELS_DIR}/${blobs[$i]}" + if ! in_container test -f "${path}"; then + echo "[FAIL] ${blobs[$i]} is not in the model store" >&2 + failed=1 + continue + fi + actual="$(in_container sha256sum "${path}" 2>/dev/null | awk '{print $1}')" + if [ "${actual}" != "${digests[$i]}" ]; then + echo "[FAIL] ${blobs[$i]} hashes to ${actual}, declared ${digests[$i]}" >&2 + failed=1 + continue + fi + echo "[OK] ${blobs[$i]} present and matching" +done + +echo "=== 3. the daemon serves every declared model ===" +mapfile -t models < <(json_items "${LMSTUDIO_EXPECTED_MODELS}") +for model in "${models[@]}"; do + attempt=1 + while [ "${attempt}" -le "${RETRIES}" ]; do + if serves_model "${model}"; then + echo "[OK] ${model} is served" + break + fi + attempt=$((attempt + 1)) + [ "${attempt}" -le "${RETRIES}" ] && sleep "${SLEEP_SECONDS}" + done + if [ "${attempt}" -gt "${RETRIES}" ]; then + echo "[FAIL] ${model} is on disk but /v1/models never listed it" >&2 + echo " the daemon did not index the store it was given at startup" >&2 + failed=1 + fi +done + +[ "${failed}" -eq 0 ] || exit 1 +echo "[OK] svc-ai-lmstudio serves every declared model from the seeded store" diff --git a/roles/svc-ai-lmstudio/meta/tests.yml b/roles/svc-ai-lmstudio/meta/tests.yml new file mode 100644 index 0000000000..712dbdf2f8 --- /dev/null +++ b/roles/svc-ai-lmstudio/meta/tests.yml @@ -0,0 +1,4 @@ +--- +cli: + container_service_key: lmstudio + timeout: 900 diff --git a/roles/svc-ai-lmstudio/meta/volumes.yml b/roles/svc-ai-lmstudio/meta/volumes.yml index ae7259ecdf..0250216548 100644 --- a/roles/svc-ai-lmstudio/meta/volumes.yml +++ b/roles/svc-ai-lmstudio/meta/volumes.yml @@ -1,17 +1,17 @@ --- -# nocheck: embedded-state-on-nfs model blobs written once on pull and read back afterwards lmstudio_data: type: volume + name: lmstudio_data + # nocheck: nfs-false-data-loss Reason: per-daemon state LM Studio rewrites in place (lms-key-2, settings.json, extensions/plugins, bin/lms, server-logs), regenerated on every start; the models it indexes live in lmstudio_models + nfs: false mounts: - service: lmstudio target: /root/.lmstudio -lmstudio_internal: +# nocheck: embedded-state-on-nfs content-addressed model blobs only; every writer fetches against the sha256 AI_LOCAL_MODELS declares and the play fails unless the store reads back at that digest, so concurrent writers of one file write identical bytes +lmstudio_models: type: volume - name: lmstudio_internal - # nocheck: nfs-false-data-loss Reason: regenerable per-daemon cache, rebuilt in full on first start against an empty volume; shared, a second daemon overwrites the auth key lms-key-2 and the first one's CLI is rejected - nfs: false - backup: false + name: lmstudio_models mounts: - service: lmstudio - target: /root/.lmstudio/.internal + target: /root/.lmstudio/models diff --git a/roles/svc-ai-lmstudio/tasks/00_core.yml b/roles/svc-ai-lmstudio/tasks/00_core.yml index 16942c364d..e29cf2a6f2 100644 --- a/roles/svc-ai-lmstudio/tasks/00_core.yml +++ b/roles/svc-ai-lmstudio/tasks/00_core.yml @@ -1,6 +1,14 @@ --- - include_tasks: utils/once/flag.yml +- name: "โฌ‡๏ธ Seed the shared model store before the daemons index it" + ansible.builtin.include_tasks: + file: utils/preseed.yml + apply: + delegate_to: "{{ LMSTUDIO_NFS_SERVER }}" + run_once: true + when: LMSTUDIO_PRESEED_ENABLED | bool + - name: "๐Ÿ”ง Setup container network for {{ application_id }}" include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/network/routine.yml') }}" vars: diff --git a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml index ac3c91c989..a4362e07d8 100644 --- a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml +++ b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml @@ -1,26 +1,10 @@ --- -- name: "๐Ÿ” Resolve {{ application_id }} hosting node + container id" - ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml') }}" - vars: - resolve_service_key: "{{ entity_name }}" - resolve_output_prefix: LMSTUDIO - resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" +- name: "๐Ÿ” Resolve every node that hosts an {{ application_id }} task" + ansible.builtin.include_tasks: utils/nodes.yml -- name: "๐ŸŒฑ Seed the models from the node cache before preloading" - ansible.builtin.include_tasks: - file: utils/seed.yml - apply: - delegate_to: "{{ LMSTUDIO_HOST_NODE }}" - -- name: "โฌ‡๏ธ Preload the LM Studio models" - block: - - name: "โฌ‡๏ธ Preload the LM Studio models (first pass)" - ansible.builtin.include_tasks: utils/preload.yml - rescue: - - name: "๐Ÿ Re-raise the preload failure unchanged in swarm" # nocheck: rescue-recovery a timed-out transfer leaves the daemon reporting 0.00% for every further pull, and restarting the container it owns is the only lever the role has, though CI job 105558224377 shows a restart returning rc 0 without clearing it; the generic capture cannot even try, so swarm re-raises here instead, because under any_errors_fatal a rescued host keeps running while every other host is already marked failed, and compose ends the block by preloading again so an unrecovered failure still re-raises - ansible.builtin.fail: - msg: "{{ ansible_failed_result.msg | default('LM Studio preload failed') }}" - when: DEPLOYMENT_MODE == 'swarm' - - - name: "โ™ป๏ธ Recover the wedged transfer and preload again" - ansible.builtin.include_tasks: utils/recover.yml +- name: "โฌ‡๏ธ Preload LM Studio on each hosting node" + ansible.builtin.include_tasks: utils/preload/node.yml + loop: "{{ LMSTUDIO_HOST_NODES }}" + loop_control: + loop_var: lmstudio_node + label: "{{ lmstudio_node }}" diff --git a/roles/svc-ai-lmstudio/tasks/utils/cid.yml b/roles/svc-ai-lmstudio/tasks/utils/cid.yml new file mode 100644 index 0000000000..e29b8fafba --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/cid.yml @@ -0,0 +1,22 @@ +--- +- name: "๐Ÿ” Resolve the LM Studio container on '{{ lmstudio_node }}'" + ansible.builtin.shell: | + {% if DEPLOYMENT_MODE == 'swarm' %} + container ps \ + --filter "label=com.docker.swarm.service.name={{ lmstudio_service_name }}" \ + --filter status=running \ + --format '{{ "{{.ID}}" }}' | head -1 + {% else %} + echo {{ LMSTUDIO_CONTAINER | quote }} + {% endif %} + register: lmstudio_node_cid + changed_when: false + retries: 30 + delay: 2 + until: DEPLOYMENT_MODE == 'compose' or (lmstudio_node_cid.stdout | trim) | length > 0 + delegate_to: "{{ lmstudio_node }}" + +- name: "๐Ÿ”ง Target '{{ lmstudio_node }}' for the next LM Studio step" + ansible.builtin.set_fact: + LMSTUDIO_HOST_NODE: "{{ lmstudio_node }}" + LMSTUDIO_LOCAL_CID: "{{ lmstudio_node_cid.stdout | trim }}" diff --git a/roles/svc-ai-lmstudio/tasks/utils/nodes.yml b/roles/svc-ai-lmstudio/tasks/utils/nodes.yml new file mode 100644 index 0000000000..b3ee1a6726 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/nodes.yml @@ -0,0 +1,29 @@ +--- +- name: "๐Ÿ”ง Resolve the swarm service name of '{{ entity_name }}'" + ansible.builtin.set_fact: + lmstudio_service_name: "{{ lookup('container_service', application_id, entity_name) if DEPLOYMENT_MODE == 'swarm' else '' }}" + +- name: "โณ Wait until every LM Studio task has converged" + ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/swarm/wait_converged.yml') }}" + vars: + wait_service_key: "{{ entity_name }}" + wait_service_name: "{{ lmstudio_service_name }}" + +- name: "๐Ÿ” List every node that runs an LM Studio task" + ansible.builtin.shell: | + container service ps {{ lmstudio_service_name | quote }} \ + --filter desired-state=running \ + --format '{{ "{{.Node}}" }}' | sort -u + register: lmstudio_node_scan + changed_when: false + retries: 30 + delay: 2 + until: (lmstudio_node_scan.stdout | trim) | length > 0 + delegate_to: "{{ STACK_HOST }}" + when: DEPLOYMENT_MODE == 'swarm' + +- name: "๐Ÿ”ง Set the LM Studio hosting nodes" + ansible.builtin.set_fact: + LMSTUDIO_HOST_NODES: >- + {{ (lmstudio_node_scan.stdout_lines | map('trim') | select | list) + if DEPLOYMENT_MODE == 'swarm' else [inventory_hostname] }} diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml b/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml new file mode 100644 index 0000000000..fb48dfd0a6 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml @@ -0,0 +1,42 @@ +--- +- name: "๐Ÿ” Resolve the container before preloading on '{{ lmstudio_node }}'" + ansible.builtin.include_tasks: utils/cid.yml + +- name: "๐Ÿ—„๏ธ Measure the space the model store offers on '{{ lmstudio_node }}'" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec {{ LMSTUDIO_LOCAL_CID | quote }} df -P -B1 /root/.lmstudio/models + register: lmstudio_store_space + changed_when: false + retries: 30 + delay: 2 + until: lmstudio_store_space.rc == 0 + delegate_to: "{{ lmstudio_node }}" + when: not (LMSTUDIO_PRESEED_ENABLED | bool) + +- name: "๐Ÿ›‘ Refuse to preload models the store on '{{ lmstudio_node }}' cannot hold" + vars: + _free: "{{ lmstudio_store_space.stdout_lines[1].split()[3] | int }}" + _required: "{{ LMSTUDIO_PRELOAD_LARGEST_BYTES | int }}" + ansible.builtin.fail: + msg: >- + The model store on {{ lmstudio_node }} has + {{ (_free | int / 1024 / 1024 / 1024) | round(1) }} GiB free, less than the + {{ (_required | int / 1024 / 1024 / 1024) | round(1) }} GiB the largest + declared model needs. Shrink AI_LOCAL_MODELS in group_vars/all/16_ai.yml or + give the node more disk. + when: + - not (LMSTUDIO_PRESEED_ENABLED | bool) + - _free | int < _required | int + +- name: "โฌ‡๏ธ Preload the LM Studio models on '{{ lmstudio_node }}'" + block: + - name: "โฌ‡๏ธ Preload the LM Studio models (first pass)" + ansible.builtin.include_tasks: utils/preload/pull.yml + rescue: + - name: "๐Ÿ Re-raise the preload failure unchanged in swarm" # nocheck: rescue-recovery a timed-out transfer leaves the daemon reporting 0.00% for every further pull, and restarting the container it owns is the only lever the role has, though CI job 105558224377 shows a restart returning rc 0 without clearing it; the generic capture cannot even try, so swarm re-raises here instead, because under any_errors_fatal a rescued host keeps running while every other host is already marked failed, and compose ends the block by preloading again so an unrecovered failure still re-raises + ansible.builtin.fail: + msg: "{{ ansible_failed_result.msg | default('LM Studio preload failed') }}" + when: DEPLOYMENT_MODE == 'swarm' + + - name: "โ™ป๏ธ Recover the wedged transfer and preload again" + ansible.builtin.include_tasks: utils/preload/recover.yml diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload.yml b/roles/svc-ai-lmstudio/tasks/utils/preload/pull.yml similarity index 100% rename from roles/svc-ai-lmstudio/tasks/utils/preload.yml rename to roles/svc-ai-lmstudio/tasks/utils/preload/pull.yml diff --git a/roles/svc-ai-lmstudio/tasks/utils/recover.yml b/roles/svc-ai-lmstudio/tasks/utils/preload/recover.yml similarity index 95% rename from roles/svc-ai-lmstudio/tasks/utils/recover.yml rename to roles/svc-ai-lmstudio/tasks/utils/preload/recover.yml index cea5918d25..e8f526456f 100644 --- a/roles/svc-ai-lmstudio/tasks/utils/recover.yml +++ b/roles/svc-ai-lmstudio/tasks/utils/preload/recover.yml @@ -25,4 +25,4 @@ resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" - name: "โฌ‡๏ธ Preload the LM Studio models (after the restart)" - ansible.builtin.include_tasks: utils/preload.yml + ansible.builtin.include_tasks: utils/preload/pull.yml diff --git a/roles/svc-ai-lmstudio/tasks/utils/preseed.yml b/roles/svc-ai-lmstudio/tasks/utils/preseed.yml new file mode 100644 index 0000000000..49fa570841 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/preseed.yml @@ -0,0 +1,118 @@ +--- +- name: "๐Ÿ“ Create the shared model store before the daemons start" + ansible.builtin.file: + path: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}" + state: directory + mode: "0755" + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + +- name: "๐Ÿ“ Create the staging directory beside the model store" + ansible.builtin.file: + path: "{{ LMSTUDIO_SHARED_STAGING_PATH }}" + state: directory + mode: "0755" + +- name: "๐Ÿ” Read what the shared model store already holds" + ansible.builtin.stat: + path: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}/{{ model.file }}" + checksum_algorithm: sha256 + register: lmstudio_present + changed_when: false + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + +- name: "๐Ÿ—„๏ธ Measure the free space the shared model store offers" + ansible.builtin.command: "df -P -B1 {{ LMSTUDIO_SHARED_MODELS_PATH | quote }}" + register: lmstudio_export_space + changed_when: false + +- name: "๐Ÿ›‘ Refuse to fetch models the shared store cannot hold" + vars: + _free: "{{ lmstudio_export_space.stdout_lines[1].split()[3] | int }}" + _absent: >- + {{ lmstudio_present.results + | rejectattr('stat.exists') + | map(attribute='model.bytes') | map('int') | sum }} + _required: "{{ (_absent | int) + (LMSTUDIO_PRELOAD_LARGEST_BYTES | int) }}" + ansible.builtin.fail: + msg: >- + The shared model store on {{ LMSTUDIO_NFS_SERVER }} has + {{ (_free | int / 1024 / 1024 / 1024) | round(1) }} GiB free but needs + {{ (_required | int / 1024 / 1024 / 1024) | round(1) }} GiB: it is missing + {{ (_absent | int / 1024 / 1024 / 1024) | round(1) }} GiB of declared models, + plus {{ (LMSTUDIO_PRELOAD_LARGEST_BYTES | int / 1024 / 1024 / 1024) | round(1) }} + GiB for the largest one while it stages. Shrink AI_LOCAL_MODELS in + group_vars/all/16_ai.yml or grow the export. + when: _free | int < _required | int + +- name: "โฌ‡๏ธ Fetch the models into the shared store" + get_url_retry: + url: "{{ model.url }}" + dest: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}/{{ model.file }}" + tmp_dest: "{{ LMSTUDIO_SHARED_STAGING_PATH }}" + checksum: "sha256:{{ model.sha256 }}" + mode: "0644" + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + register: lmstudio_fetch + changed_when: false + failed_when: false + +- name: "๐Ÿชž Fetch from a mirror when the source did not deliver" + get_url_retry: + url: "{{ item.1 }}" + dest: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ item.0.repo }}/{{ item.0.file }}" + tmp_dest: "{{ LMSTUDIO_SHARED_STAGING_PATH }}" + checksum: "sha256:{{ item.0.sha256 }}" + mode: "0644" + loop: "{{ LMSTUDIO_PRELOAD_MODELS | subelements('mirrors', skip_missing=True) }}" + loop_control: + label: "{{ item.0.file }} from {{ item.1 }}" + register: lmstudio_fetch_mirror + changed_when: false + failed_when: false + +- name: "๐Ÿ” Read back what the shared model store now holds" + ansible.builtin.stat: + path: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}/{{ model.file }}" + checksum_algorithm: sha256 + register: lmstudio_seeded + changed_when: false + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + +- name: "๐Ÿ›‘ Fail when no source delivered a declared model" + vars: + _missing: >- + {{ lmstudio_seeded.results + | rejectattr('stat.exists') + | map(attribute='model.file') | list }} + _said: >- + {{ (lmstudio_fetch.results | map(attribute='msg') | select | list) + + (lmstudio_fetch_mirror.results | default([]) | map(attribute='msg') | select | list) }} + ansible.builtin.fail: + msg: >- + The shared model store on {{ LMSTUDIO_NFS_SERVER }} is missing + {{ _missing | join(', ') }}. What the sources answered: + {{ _said | join(' | ') }} + when: _missing | length > 0 + +- name: "๐Ÿ›‘ Fail when a seeded model does not match its checksum" + ansible.builtin.fail: + msg: >- + {{ item.model.file }} in the shared model store on {{ LMSTUDIO_NFS_SERVER }} + hashes to {{ item.stat.checksum }}, but AI_LOCAL_MODELS declares + {{ item.model.sha256 }}. + loop: "{{ lmstudio_seeded.results | selectattr('stat.exists') | list }}" + loop_control: + label: "{{ item.model.file }}" + when: item.stat.checksum != item.model.sha256 diff --git a/roles/svc-ai-lmstudio/tasks/utils/seed.yml b/roles/svc-ai-lmstudio/tasks/utils/seed.yml deleted file mode 100644 index 41d2501a3c..0000000000 --- a/roles/svc-ai-lmstudio/tasks/utils/seed.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -- name: "๐Ÿ“ Create the model cache directory on the hosting node" - ansible.builtin.file: - path: "{{ LMSTUDIO_MODEL_CACHE_DIR }}" - state: directory - mode: "0755" - -- name: "๐Ÿ—„๏ธ Measure the free space the models have to fit into" - ansible.builtin.command: "df -P -B1 {{ LMSTUDIO_MODEL_CACHE_DIR | quote }}" - register: lmstudio_cache_space - changed_when: false - -- name: "๐Ÿ›‘ Refuse to fetch models the hosting node cannot hold" - vars: - _free: "{{ lmstudio_cache_space.stdout_lines[1].split()[3] | int }}" - _required: "{{ LMSTUDIO_PRELOAD_REQUIRED_BYTES | int }}" - ansible.builtin.fail: - msg: >- - The declared LM Studio models need {{ (_required | int / 1024 / 1024 / 1024) - | round(1) }} GiB on {{ LMSTUDIO_HOST_NODE }} but only {{ (_free | int / 1024 - / 1024 / 1024) | round(1) }} GiB is free. That is - {{ LMSTUDIO_PRELOAD_COPIES_ON_DISK }} times the - {{ (LMSTUDIO_PRELOAD_BYTES | int / 1024 / 1024 / 1024) | round(1) }} GiB the - models declare, because the node cache, the staging copy inside the container - and the models volume all exist at once during the import. Shrink - AI_LOCAL_MODELS in group_vars/all/16_ai.yml or give the node more disk. - when: _free | int < _required | int - -- name: "โฌ‡๏ธ Warm the model cache on the node, best effort" - get_url_retry: - url: "{{ model.source }}/resolve/main/{{ model.file }}" - dest: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ model.file }}" - checksum: "sha256:{{ model.sha256 }}" - mode: "0644" - loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" - loop_control: - loop_var: model - label: "{{ model.file }}" - changed_when: false - failed_when: false - -- name: "๐Ÿชž Warm the model cache from a mirror when the source stayed silent" - get_url_retry: - url: "{{ item.1 }}/resolve/main/{{ item.0.file }}" - dest: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ item.0.file }}" - checksum: "sha256:{{ item.0.sha256 }}" - mode: "0644" - loop: "{{ LMSTUDIO_PRELOAD_MODELS | subelements('mirrors', skip_missing=True) }}" - loop_control: - label: "{{ item.0.file }} from {{ item.1 }}" - changed_when: false - failed_when: false - -- name: "๐Ÿ“‹ Hand the warmed model to LM Studio" - ansible.builtin.script: shell/import.sh - environment: - LMSTUDIO_LOCAL_CID: "{{ LMSTUDIO_LOCAL_CID }}" - MODEL_FILE: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ model.file }}" - MODEL_USER_REPO: "{{ model.source | regex_replace('^https://huggingface\\.co/', '') }}" - MODELS_DIR: "{{ LMSTUDIO_MODELS_DIR }}" - loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" - loop_control: - loop_var: model - label: "{{ model.file }}" - changed_when: false diff --git a/roles/svc-ai-lmstudio/templates/test.env.j2 b/roles/svc-ai-lmstudio/templates/test.env.j2 new file mode 100644 index 0000000000..4913d84187 --- /dev/null +++ b/roles/svc-ai-lmstudio/templates/test.env.j2 @@ -0,0 +1,5 @@ +LMSTUDIO_CONTAINER={{ CLI_LOCAL_CID }} +LMSTUDIO_EXPECTED_BLOBS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='repo') | zip(LMSTUDIO_PRELOAD_MODELS | map(attribute='file')) | map('join', '/') | list | to_json }}' +LMSTUDIO_EXPECTED_DIGESTS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='sha256') | list | to_json }}' +LMSTUDIO_EXPECTED_MODELS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='name') | list | to_json }}' +LMSTUDIO_PORT={{ LMSTUDIO_PORT }} diff --git a/roles/svc-ai-lmstudio/vars/main.yml b/roles/svc-ai-lmstudio/vars/main.yml index 3c55763554..b55d6d4644 100644 --- a/roles/svc-ai-lmstudio/vars/main.yml +++ b/roles/svc-ai-lmstudio/vars/main.yml @@ -8,12 +8,14 @@ LMSTUDIO_PRELOAD_MODELS: "{{ lookup('config', application_id, 'services.lmstudio LMSTUDIO_PRELOAD_ASYNC_TIME: "{{ lookup('config', application_id, 'services.lmstudio.preload_async_time') }}" LMSTUDIO_PRELOAD_FLOOR_SECONDS: "{{ lookup('config', application_id, 'services.lmstudio.preload_floor_seconds') }}" LMSTUDIO_PRELOAD_MIN_BYTES_PER_SECOND: "{{ lookup('config', application_id, 'services.lmstudio.preload_min_bytes_per_second') }}" -LMSTUDIO_PRELOAD_BYTES: >- - {{ LMSTUDIO_PRELOAD_MODELS | map(attribute='bytes') | map('int') | sum }} -LMSTUDIO_PRELOAD_COPIES_ON_DISK: 3 -LMSTUDIO_PRELOAD_REQUIRED_BYTES: >- - {{ (LMSTUDIO_PRELOAD_BYTES | int) * (LMSTUDIO_PRELOAD_COPIES_ON_DISK | int) }} -LMSTUDIO_CONFIG_DIR: "{{ lookup('container', application_id, 'directories.config') }}" -LMSTUDIO_MODEL_CACHE_DIR: "{{ [ LMSTUDIO_CONFIG_DIR, 'models' ] | path_join }}" -LMSTUDIO_HOME_DIR: "{{ lookup('volume', application_id, 'lmstudio_data').target }}" -LMSTUDIO_MODELS_DIR: "{{ [ LMSTUDIO_HOME_DIR, 'models' ] | path_join }}" +LMSTUDIO_PRELOAD_LARGEST_BYTES: >- + {{ LMSTUDIO_PRELOAD_MODELS | map(attribute='bytes') | map('int') | max }} +LMSTUDIO_NFS_SERVER: "{{ (groups['svc-storage-nfs-server'] | default([]))[0] | default(inventory_hostname) }}" +LMSTUDIO_SHARED_MODELS_PATH: "{{ [ STORAGE_NFS_STATE_PATH, lookup('volume', application_id, 'lmstudio_models').name ] | path_join }}" +LMSTUDIO_SHARED_STAGING_PATH: "{{ [ STORAGE_NFS_STATE_PATH, '.lmstudio-staging' ] | path_join }}" +LMSTUDIO_PRESEED_ENABLED: >- + {{ (DEPLOYMENT_MODE == 'swarm') + and (storage.backend == 'nfs') + and ((groups['svc-storage-nfs-server'] | default([])) | length > 0) + and (lookup('volume', application_id, 'lmstudio_models').nfs is not false) + and (application_id not in lookup('roles_with_placement', 'manager')) }} diff --git a/tests/external/roles/test_ai_local_models_downloadable.py b/tests/external/roles/test_ai_local_models_downloadable.py index cc44488f16..4f013d6788 100644 --- a/tests/external/roles/test_ai_local_models_downloadable.py +++ b/tests/external/roles/test_ai_local_models_downloadable.py @@ -41,25 +41,26 @@ def _models() -> list[dict]: return [entry for entry in (entries or []) if isinstance(entry, dict)] -def _repo_path(source: str) -> str: - """Return ``user/repo`` for a HuggingFace source, else the empty string.""" - if not source.startswith(_HUGGINGFACE_PREFIX): +def _repo_path(entry: dict) -> str: + """Return the HuggingFace ``user/repo`` this entry pins, else empty. + + Args: + entry: one AI_LOCAL_MODELS declaration. + """ + url = str(entry.get("url") or "") + if not url.startswith(_HUGGINGFACE_PREFIX): return "" - return source[len(_HUGGINGFACE_PREFIX) :].strip("/") + return str(entry.get("repo") or "").strip("/") def _download_url(entry: dict) -> str: - return f"{entry['source'].rstrip('/')}/resolve/main/{entry['file']}" + return str(entry["url"]) def _mirror_urls(entry: dict) -> list[str]: - """Every extra base a deploy falls back to, in declaration order.""" + """Every extra URL a deploy falls back to, in declaration order.""" mirrors = entry.get("mirrors") or [] - return [ - f"{str(base).rstrip('/')}/resolve/main/{entry['file']}" - for base in mirrors - if isinstance(base, str) - ] + return [str(url) for url in mirrors if isinstance(url, str)] def _upstream_digests(repo_path: str) -> tuple[str, object]: @@ -133,11 +134,14 @@ class TestAiLocalModelsDownloadable(unittest.TestCase): unpinned = [] for entry in models: alias = str(entry.get("alias") or "") - source = entry.get("source") + url = entry.get("url") + repo = entry.get("repo") file_name = entry.get("file") digest = entry.get("sha256") - if not isinstance(source, str) or not urlsplit(source).scheme: - unpinned.append(f"{alias}: 'source' is not an absolute URL") + if not isinstance(url, str) or not urlsplit(url).scheme: + unpinned.append(f"{alias}: 'url' is not an absolute URL") + if not isinstance(repo, str) or len(repo.strip("/").split("/")) != 2: + unpinned.append(f"{alias}: 'repo' is not '/'") if not isinstance(file_name, str) or not file_name: unpinned.append(f"{alias}: 'file' is missing") if not isinstance(digest, str) or not _SHA256.match(digest): @@ -149,21 +153,21 @@ class TestAiLocalModelsDownloadable(unittest.TestCase): self.assertFalse( unpinned, f"{len(unpinned)} model declaration(s) in {_AI_VARS_FILE} cannot be " - f"fetched with a checksum. Each entry needs 'source', 'file' and " - f"'sha256':\n" + "\n".join(f" {line}" for line in unpinned), + f"fetched with a checksum. Each entry needs 'url', 'repo', 'file' " + f"and 'sha256':\n" + "\n".join(f" {line}" for line in unpinned), ) def test_every_pin_matches_what_upstream_serves(self) -> None: models = [ entry for entry in _models() - if isinstance(entry.get("source"), str) + if isinstance(entry.get("url"), str) and isinstance(entry.get("file"), str) and isinstance(entry.get("sha256"), str) ] self.assertTrue(models, f"{_AI_VARS_FILE} declares no pinned model") - repo_paths = [_repo_path(entry["source"]) for entry in models] + repo_paths = [_repo_path(entry) for entry in models] with concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_WORKERS) as pool: api = list(pool.map(_upstream_digests, repo_paths)) reach = list(pool.map(_reachable, [_download_url(e) for e in models])) From f10d281533f176cb06054b277735d96572929fa7 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 12:36:35 +0200 Subject: [PATCH 154/197] fix(lmstudio): read the models mount from the registry, not from memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI job 105881436871 on 0b7a8a95d: A container path declared in meta/volumes.yml is written out again. roles/svc-ai-lmstudio/tasks/utils/preload/node.yml:7: /root/.lmstudio/models (declared as 'lmstudio_models') The disk guard typed the mount target instead of deriving it. That matters more since the split: the role now carries two nested volumes, and which of them holds models/ is decided solely by meta/volumes.yml. A second copy of the path drifts the moment the mount moves, which is the drift 29015311b6 built this lint for. LMSTUDIO_MODELS_TARGET resolves it through lookup('volume', โ€ฆ).target, next to LMSTUDIO_SHARED_MODELS_PATH, which already reads .name from the same entry. The test script held the same path a second time. The lint only reads yaml and templates, so it would never have reported that copy, and it would have gone stale just as quietly. It now receives the value through test.env. Not caught locally, and the reason is worth recording: the preceding `make quality` was red on three formalities of the new test assets. I fixed those, re-ran only those three lints, and deployed. This finding sat a few lines below them in the same output. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-lmstudio/files/test/test.sh | 3 ++- roles/svc-ai-lmstudio/tasks/utils/preload/node.yml | 2 +- roles/svc-ai-lmstudio/templates/test.env.j2 | 1 + roles/svc-ai-lmstudio/vars/main.yml | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/svc-ai-lmstudio/files/test/test.sh b/roles/svc-ai-lmstudio/files/test/test.sh index fc83bfc748..c981fce35d 100644 --- a/roles/svc-ai-lmstudio/files/test/test.sh +++ b/roles/svc-ai-lmstudio/files/test/test.sh @@ -7,6 +7,7 @@ # LMSTUDIO_EXPECTED_MODELS JSON array of model names the daemon must serve # LMSTUDIO_EXPECTED_BLOBS JSON array of / paths under models/ # LMSTUDIO_EXPECTED_DIGESTS JSON array of sha256, aligned with the blobs +# LMSTUDIO_MODELS_DIR the models mount inside the container # RETRIES attempts (default 30) # SLEEP_SECONDS wait between (default 10) @@ -14,7 +15,7 @@ set -uo pipefail RETRIES="${RETRIES:-30}" SLEEP_SECONDS="${SLEEP_SECONDS:-10}" -MODELS_DIR=/root/.lmstudio/models +MODELS_DIR="${LMSTUDIO_MODELS_DIR}" [ -n "${LMSTUDIO_CONTAINER}" ] || { echo "[FATAL] LMSTUDIO_CONTAINER unset; the harness did not resolve a container" >&2 diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml b/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml index fb48dfd0a6..b89e50e2e5 100644 --- a/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml +++ b/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml @@ -4,7 +4,7 @@ - name: "๐Ÿ—„๏ธ Measure the space the model store offers on '{{ lmstudio_node }}'" ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node - container exec {{ LMSTUDIO_LOCAL_CID | quote }} df -P -B1 /root/.lmstudio/models + container exec {{ LMSTUDIO_LOCAL_CID | quote }} df -P -B1 {{ LMSTUDIO_MODELS_TARGET | quote }} register: lmstudio_store_space changed_when: false retries: 30 diff --git a/roles/svc-ai-lmstudio/templates/test.env.j2 b/roles/svc-ai-lmstudio/templates/test.env.j2 index 4913d84187..019aeb6e20 100644 --- a/roles/svc-ai-lmstudio/templates/test.env.j2 +++ b/roles/svc-ai-lmstudio/templates/test.env.j2 @@ -2,4 +2,5 @@ LMSTUDIO_CONTAINER={{ CLI_LOCAL_CID }} LMSTUDIO_EXPECTED_BLOBS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='repo') | zip(LMSTUDIO_PRELOAD_MODELS | map(attribute='file')) | map('join', '/') | list | to_json }}' LMSTUDIO_EXPECTED_DIGESTS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='sha256') | list | to_json }}' LMSTUDIO_EXPECTED_MODELS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='name') | list | to_json }}' +LMSTUDIO_MODELS_DIR={{ LMSTUDIO_MODELS_TARGET }} LMSTUDIO_PORT={{ LMSTUDIO_PORT }} diff --git a/roles/svc-ai-lmstudio/vars/main.yml b/roles/svc-ai-lmstudio/vars/main.yml index b55d6d4644..8314305c99 100644 --- a/roles/svc-ai-lmstudio/vars/main.yml +++ b/roles/svc-ai-lmstudio/vars/main.yml @@ -10,6 +10,7 @@ LMSTUDIO_PRELOAD_FLOOR_SECONDS: "{{ lookup('config', application_id, 'services.l LMSTUDIO_PRELOAD_MIN_BYTES_PER_SECOND: "{{ lookup('config', application_id, 'services.lmstudio.preload_min_bytes_per_second') }}" LMSTUDIO_PRELOAD_LARGEST_BYTES: >- {{ LMSTUDIO_PRELOAD_MODELS | map(attribute='bytes') | map('int') | max }} +LMSTUDIO_MODELS_TARGET: "{{ lookup('volume', application_id, 'lmstudio_models').target }}" LMSTUDIO_NFS_SERVER: "{{ (groups['svc-storage-nfs-server'] | default([]))[0] | default(inventory_hostname) }}" LMSTUDIO_SHARED_MODELS_PATH: "{{ [ STORAGE_NFS_STATE_PATH, lookup('volume', application_id, 'lmstudio_models').name ] | path_join }}" LMSTUDIO_SHARED_STAGING_PATH: "{{ [ STORAGE_NFS_STATE_PATH, '.lmstudio-staging' ] | path_join }}" From d19b4b5f5a74223b2e8bdc2e69f804976b9986ac Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 16:35:48 +0200 Subject: [PATCH 155/197] Release version 14.0.0 --- CHANGELOG.md | 143 ++++++++++++++++++++++++++ packaging/arch/PKGBUILD | 2 +- packaging/debian/changelog | 145 +++++++++++++++++++++++++++ packaging/fedora/infinito-nexus.spec | 144 +++++++++++++++++++++++++- pyproject.toml | 2 +- 5 files changed, 433 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ccaf1302e..2c523b9e14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,148 @@ # Changelog +## [14.0.0] - 2026-09-19 + +**For Users** + +* **AI assistance inside the applications you already run.** A gateway role, + *svc-ai-litellm*, fronts OpenAI, Anthropic and OpenRouter from declared keys, with + *svc-ai-ollama* and the new *svc-ai-lmstudio* serving local models from one shared + catalogue, so inference can stay inside the deployment. Sixteen applications use it for + their own built-in AI features instead of calling a vendor directly: WordPress AI Engine, + MediaWiki AIEditingAssistant, Moodle's AI subsystem, Zammad Smart Assist, the xWiki LLM + extension, Discourse AI, the Matrix ChatGPT bridge, Nextcloud, Mattermost Agents, n8n, + Open WebUI and Flowise. The gateway's admin UI signs in through Keycloak. See + [svc-ai-litellm](roles/svc-ai-litellm/README.md) and + [svc-ai-lmstudio](roles/svc-ai-lmstudio/README.md). + +* **Your tools reachable from your assistants.** About twenty applications now speak MCP. + Gitea, GitLab, Mattermost, Nextcloud, n8n, Baserow, Jenkins, Home Assistant, Moodle, + Snipe-IT and WordPress serve their own endpoints; ten more get one through a locked-down + sidecar. Open WebUI, Flowise and the agents discover what the deployment offers and + register it automatically. Access is bounded rather than assumed: blocked paths are + declared, every served path is gated, tools a provider calls mutating are withheld, + reader and writer roles are separate, and each provider authenticates callers as its own + service account. See [svc-ai-mcp-adapter](roles/svc-ai-mcp-adapter/README.md). + +* **Agents as employees, with a kernel between them and the host.** *web-app-hermes* and + *web-app-openclaw* deploy agents behind the single sign-on gate, wired to the gateway's + models and to the MCP servers discovery hands them, each proving on deploy that it can + actually reach them. *svc-virt-kata* runs that tier under a kernel-isolating container + runtime, and *svc-ai-robot* embodies an agent on a dedicated device. Every sandboxed + agent acts as its own platform account rather than borrowing yours. See + [web-app-hermes](roles/web-app-hermes/README.md), + [web-app-openclaw](roles/web-app-openclaw/README.md) and + [svc-virt-kata](roles/svc-virt-kata/README.md). + +* **Home automation joins the platform.** *web-app-homeassistant* deploys the automation + hub, exposes it as an MCP server so agents can drive it, persists the token it mints, and + is reachable over tor. See [web-app-homeassistant](roles/web-app-homeassistant/README.md). + +* **Changing a password changes it everywhere.** A user declares how its password must be + generated, the inventory pins one for every declared user, a new generator produces + API-key shapes, the deploy rotates credentials between its two passes, and a reset command + exists. Applications that persist an administrator password at install now realign it when + the declared one changes: Nextcloud, Moodle, EspoCRM, Jellyfin, Listmonk, Discourse, + Checkmk, xWiki, n8n, Matrix and the dedicated MariaDB and Postgres engines. Checkmk keeps + a 401 fatal so a wrong password cannot lock the account out. + +* **Onions, firewall and name resolution.** *svc-net-firewall* owns the packet filter as one + nftables table per role. Tor carries mail to a .onion in both deploy modes, rejects forged + sources on the guarded ports, and binds its dnsmasq listeners statically so a failed + re-enumeration can no longer drop every one of them. The deployment declares its own + address space once, the egress range moved out of the Docker pool, and the container + resolver follows the live bridge instead of a guessed address. Every host-bound port + requires an explicit onion decision. See + [svc-net-firewall](roles/svc-net-firewall/README.md). + +* **Application fixes you will notice.** Nextcloud installs an OnlyOffice release upstream + actually published, keeps social login working over tor, and makes its OIDC account an + administrator. Element shows invites again through the new room list. Moodle survives the + container roll during installation. Mattermost keeps its configuration in Postgres instead + of on a shared volume, and its plugin tree is per node. GitLab creates the default + organization a headless install never gets. Jellyfin waits for health before driving the + setup wizard. Keycloak registers redirect URIs only for the apps a play deploys, keeps the + allowed-origins claim out of the access token, and no longer caches LDAP federation past a + fresh grant. The platform's own health mail no longer eats the sender quota. + +* **Operating a deployment.** New make targets reach a single deployed application: + *compose-app-exec*, *compose-app-logs* and *compose-app-restart*. Diagnostics collect the + logs an application writes to a file, including from busybox containers. Health probes use + a fast start cadence with a window that outlasts a real boot, so a slow starter is no + longer reaped. The CSP health check accepts a status code a vhost declares instead of + dropping the vhost, and gives onion vhosts a navigation budget Tor can meet. + +**For Developers** + +* **MCP is declared, not wired.** A role states an *mcp* block in *meta/services.yml*, and + the vocabulary, discovery and rendering layer derives endpoints, clients, credentials and + network edges from it. A client-provider pairing is declared once, on the provider; + *sys-svc-mcp-reconcile* converges clients from the complete provider set; a refused pairing + is expressed as off rather than gated at the call site; and *svc-ai-mcp-adapter* is a + reusable per-provider sidecar rather than a standalone application. The contract is proven + on deploy, including refusal on *initialize* and the tool inventory itself. The design, + its delegation rules and a machine-checkable audit live under + [docs/contributing/design/role/services/mcp.md](docs/contributing/design/role/services/mcp.md). + +* **Registry and variant changes.** A variant can dictate the configuration of the providers + it pulls in, and an MCP round decides for itself what it carries, which is the breaking + change in this release. Fifteen moving pins gained names the registry can order, every MCP + provider is registered as a shared service, *meta/schema.yml* finished its rename to + *meta/secrets.yml*, role scripts are sorted into per-language folders, and *meta/addons* is + now an accepted declaration site for pins and plugins. + +* **Lints that fail closed.** Five properties that used to pass silently now fail. The PHP + and Ruby that roles ship are syntax-checked, and so are shell scripts that ship as Jinja + templates. *sys-service* owns every systemd unit a role installs. Healthcheck timings are + judged against what the base image was measured to hold, a variable-free healthcheck + belongs in *meta/services.yml*, an unenforceable NFS state layout fails at author time, a + sandboxed role cannot be pinned to the manager, credentials are kept out of volumes + declared non-secret, *no_log* is required on every task carrying one and forbidden in the + test roles, and a JavaScript default for an unset environment variable is rejected. + +* **Pins that cannot age unnoticed.** Image pins are classified and pull references built + from the class, a moving tag declares itself with the marker it already carries, and a + role's images move as one release or not at all. The updater reads addon declarations, + orders vendor patch counters and five-component versions, reads a release line written + before the number as a family, and now also watches pinned pip requirements and Ansible + collections. Seven roles moved off the Debian codename base onto slim, and one Ubuntu + mirror list with failover, a short timeout and a health test serves both the package cache + and the image builds. *js-yaml* was lifted to 4.3.2 for GHSA-2883-xcg3-v3hh. + +* **CI selects, resumes and proves.** A fourth deploy chunk stops a short priority chunk + stranding budget, a manual run can set the chunk size the formula would derive, and a + retrigger can override the chunk gate. Retriggering works by role and by what never got a + verdict, resumes behind the green window rather than the deployed one, retires suites the + source run already passed, and cancels the run before retriggering its failed roles. Each + role's README instructions are replayed on its smallest deploy row, in the mode its + *services.yml* declares. Distro images build concurrently on one runner, the matrix job + folds into the image job, and the deploy chunks no longer wait on the DNS test. + +* **Swarm, measured rather than assumed.** The deploy waits for its own update instead of + guessing after it, refuses to call a stack converged while it is still updating, survives a + task that failed and came back, reports what converged rather than only what did not, and + says why the convergence probe refused. Placement is constrained to a role's declared + platform arch, redis and memcached sidecars get a deploy block, and a run-once service no + longer strands the stack deploy waiting for a replica it never has. The rescue dump + captures the contents of the state volumes. In the test lab, workers are labelled + kata-capable only when the app is sandboxed, NFS detaches before the node containers die, + and the sandbox tier gets a second node. + +* **Faster gates.** Per-host setup runs once per host instead of once per application, a + role's volumes are answered from its own file instead of a whole-tree build, the Keycloak + RBAC root group resolves once rather than once per app, the Playwright fixture copy the + tree copy already made is dropped, logout helpers wait for a control instead of sleeping, + and the OIDC login helper stops waiting out probes that cannot match. Onion timeouts scale + by tor flavor instead of a flat multiplier, and every Playwright timeout has one source. + +* **Documentation and agent instructions.** Six new requirement documents cover the LLM + gateway and model backends, agent employees, Home Assistant, the robot role, the MCP proxy + expansion and native AI gateway integration. The MCP design docs are nested under their own + folder with an audit the lints can check. The role README template asks the reader for a + domain the deployment can resolve, enforces consistent Credits attribution, and drops the em + dash. Documentation generators are gathered under *cli/build/docs*, and a new CLI edits role + bonds in a matrix instead of hunting through files. + ## [13.0.0] - 2026-08-26 * **Tor as a first-class deployment axis.** The new *svc-net-tor* role routes onion diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD index 16e17e6341..04fdd9b70e 100644 --- a/packaging/arch/PKGBUILD +++ b/packaging/arch/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: Kevin Veen-Birkenbach pkgname=infinito-nexus -pkgver=13.0.0 +pkgver=14.0.0 pkgrel=1 pkgdesc="Meta package for Infinito.Nexus host dependencies (dev + CI)." arch=('any') diff --git a/packaging/debian/changelog b/packaging/debian/changelog index f42e064389..62aa994174 100644 --- a/packaging/debian/changelog +++ b/packaging/debian/changelog @@ -1,3 +1,148 @@ +infinito-nexus (14.0.0-1) unstable; urgency=medium + + **For Users** + + * **AI assistance inside the applications you already run.** A gateway role, + *svc-ai-litellm*, fronts OpenAI, Anthropic and OpenRouter from declared keys, with + *svc-ai-ollama* and the new *svc-ai-lmstudio* serving local models from one shared + catalogue, so inference can stay inside the deployment. Sixteen applications use it for + their own built-in AI features instead of calling a vendor directly: WordPress AI Engine, + MediaWiki AIEditingAssistant, Moodle's AI subsystem, Zammad Smart Assist, the xWiki LLM + extension, Discourse AI, the Matrix ChatGPT bridge, Nextcloud, Mattermost Agents, n8n, + Open WebUI and Flowise. The gateway's admin UI signs in through Keycloak. See + [svc-ai-litellm](roles/svc-ai-litellm/README.md) and + [svc-ai-lmstudio](roles/svc-ai-lmstudio/README.md). + + * **Your tools reachable from your assistants.** About twenty applications now speak MCP. + Gitea, GitLab, Mattermost, Nextcloud, n8n, Baserow, Jenkins, Home Assistant, Moodle, + Snipe-IT and WordPress serve their own endpoints; ten more get one through a locked-down + sidecar. Open WebUI, Flowise and the agents discover what the deployment offers and + register it automatically. Access is bounded rather than assumed: blocked paths are + declared, every served path is gated, tools a provider calls mutating are withheld, + reader and writer roles are separate, and each provider authenticates callers as its own + service account. See [svc-ai-mcp-adapter](roles/svc-ai-mcp-adapter/README.md). + + * **Agents as employees, with a kernel between them and the host.** *web-app-hermes* and + *web-app-openclaw* deploy agents behind the single sign-on gate, wired to the gateway's + models and to the MCP servers discovery hands them, each proving on deploy that it can + actually reach them. *svc-virt-kata* runs that tier under a kernel-isolating container + runtime, and *svc-ai-robot* embodies an agent on a dedicated device. Every sandboxed + agent acts as its own platform account rather than borrowing yours. See + [web-app-hermes](roles/web-app-hermes/README.md), + [web-app-openclaw](roles/web-app-openclaw/README.md) and + [svc-virt-kata](roles/svc-virt-kata/README.md). + + * **Home automation joins the platform.** *web-app-homeassistant* deploys the automation + hub, exposes it as an MCP server so agents can drive it, persists the token it mints, and + is reachable over tor. See [web-app-homeassistant](roles/web-app-homeassistant/README.md). + + * **Changing a password changes it everywhere.** A user declares how its password must be + generated, the inventory pins one for every declared user, a new generator produces + API-key shapes, the deploy rotates credentials between its two passes, and a reset command + exists. Applications that persist an administrator password at install now realign it when + the declared one changes: Nextcloud, Moodle, EspoCRM, Jellyfin, Listmonk, Discourse, + Checkmk, xWiki, n8n, Matrix and the dedicated MariaDB and Postgres engines. Checkmk keeps + a 401 fatal so a wrong password cannot lock the account out. + + * **Onions, firewall and name resolution.** *svc-net-firewall* owns the packet filter as one + nftables table per role. Tor carries mail to a .onion in both deploy modes, rejects forged + sources on the guarded ports, and binds its dnsmasq listeners statically so a failed + re-enumeration can no longer drop every one of them. The deployment declares its own + address space once, the egress range moved out of the Docker pool, and the container + resolver follows the live bridge instead of a guessed address. Every host-bound port + requires an explicit onion decision. See + [svc-net-firewall](roles/svc-net-firewall/README.md). + + * **Application fixes you will notice.** Nextcloud installs an OnlyOffice release upstream + actually published, keeps social login working over tor, and makes its OIDC account an + administrator. Element shows invites again through the new room list. Moodle survives the + container roll during installation. Mattermost keeps its configuration in Postgres instead + of on a shared volume, and its plugin tree is per node. GitLab creates the default + organization a headless install never gets. Jellyfin waits for health before driving the + setup wizard. Keycloak registers redirect URIs only for the apps a play deploys, keeps the + allowed-origins claim out of the access token, and no longer caches LDAP federation past a + fresh grant. The platform's own health mail no longer eats the sender quota. + + * **Operating a deployment.** New make targets reach a single deployed application: + *compose-app-exec*, *compose-app-logs* and *compose-app-restart*. Diagnostics collect the + logs an application writes to a file, including from busybox containers. Health probes use + a fast start cadence with a window that outlasts a real boot, so a slow starter is no + longer reaped. The CSP health check accepts a status code a vhost declares instead of + dropping the vhost, and gives onion vhosts a navigation budget Tor can meet. + + **For Developers** + + * **MCP is declared, not wired.** A role states an *mcp* block in *meta/services.yml*, and + the vocabulary, discovery and rendering layer derives endpoints, clients, credentials and + network edges from it. A client-provider pairing is declared once, on the provider; + *sys-svc-mcp-reconcile* converges clients from the complete provider set; a refused pairing + is expressed as off rather than gated at the call site; and *svc-ai-mcp-adapter* is a + reusable per-provider sidecar rather than a standalone application. The contract is proven + on deploy, including refusal on *initialize* and the tool inventory itself. The design, + its delegation rules and a machine-checkable audit live under + [docs/contributing/design/role/services/mcp.md](docs/contributing/design/role/services/mcp.md). + + * **Registry and variant changes.** A variant can dictate the configuration of the providers + it pulls in, and an MCP round decides for itself what it carries, which is the breaking + change in this release. Fifteen moving pins gained names the registry can order, every MCP + provider is registered as a shared service, *meta/schema.yml* finished its rename to + *meta/secrets.yml*, role scripts are sorted into per-language folders, and *meta/addons* is + now an accepted declaration site for pins and plugins. + + * **Lints that fail closed.** Five properties that used to pass silently now fail. The PHP + and Ruby that roles ship are syntax-checked, and so are shell scripts that ship as Jinja + templates. *sys-service* owns every systemd unit a role installs. Healthcheck timings are + judged against what the base image was measured to hold, a variable-free healthcheck + belongs in *meta/services.yml*, an unenforceable NFS state layout fails at author time, a + sandboxed role cannot be pinned to the manager, credentials are kept out of volumes + declared non-secret, *no_log* is required on every task carrying one and forbidden in the + test roles, and a JavaScript default for an unset environment variable is rejected. + + * **Pins that cannot age unnoticed.** Image pins are classified and pull references built + from the class, a moving tag declares itself with the marker it already carries, and a + role's images move as one release or not at all. The updater reads addon declarations, + orders vendor patch counters and five-component versions, reads a release line written + before the number as a family, and now also watches pinned pip requirements and Ansible + collections. Seven roles moved off the Debian codename base onto slim, and one Ubuntu + mirror list with failover, a short timeout and a health test serves both the package cache + and the image builds. *js-yaml* was lifted to 4.3.2 for GHSA-2883-xcg3-v3hh. + + * **CI selects, resumes and proves.** A fourth deploy chunk stops a short priority chunk + stranding budget, a manual run can set the chunk size the formula would derive, and a + retrigger can override the chunk gate. Retriggering works by role and by what never got a + verdict, resumes behind the green window rather than the deployed one, retires suites the + source run already passed, and cancels the run before retriggering its failed roles. Each + role's README instructions are replayed on its smallest deploy row, in the mode its + *services.yml* declares. Distro images build concurrently on one runner, the matrix job + folds into the image job, and the deploy chunks no longer wait on the DNS test. + + * **Swarm, measured rather than assumed.** The deploy waits for its own update instead of + guessing after it, refuses to call a stack converged while it is still updating, survives a + task that failed and came back, reports what converged rather than only what did not, and + says why the convergence probe refused. Placement is constrained to a role's declared + platform arch, redis and memcached sidecars get a deploy block, and a run-once service no + longer strands the stack deploy waiting for a replica it never has. The rescue dump + captures the contents of the state volumes. In the test lab, workers are labelled + kata-capable only when the app is sandboxed, NFS detaches before the node containers die, + and the sandbox tier gets a second node. + + * **Faster gates.** Per-host setup runs once per host instead of once per application, a + role's volumes are answered from its own file instead of a whole-tree build, the Keycloak + RBAC root group resolves once rather than once per app, the Playwright fixture copy the + tree copy already made is dropped, logout helpers wait for a control instead of sleeping, + and the OIDC login helper stops waiting out probes that cannot match. Onion timeouts scale + by tor flavor instead of a flat multiplier, and every Playwright timeout has one source. + + * **Documentation and agent instructions.** Six new requirement documents cover the LLM + gateway and model backends, agent employees, Home Assistant, the robot role, the MCP proxy + expansion and native AI gateway integration. The MCP design docs are nested under their own + folder with an audit the lints can check. The role README template asks the reader for a + domain the deployment can resolve, enforces consistent Credits attribution, and drops the em + dash. Documentation generators are gathered under *cli/build/docs*, and a new CLI edits role + bonds in a matrix instead of hunting through files. + + -- Kevin Veen-Birkenbach Sat, 19 Sep 2026 16:35:48 +0200 + infinito-nexus (13.0.0-1) unstable; urgency=medium * **Tor as a first-class deployment axis.** The new *svc-net-tor* role routes onion diff --git a/packaging/fedora/infinito-nexus.spec b/packaging/fedora/infinito-nexus.spec index 51e99a8f89..675af778d9 100644 --- a/packaging/fedora/infinito-nexus.spec +++ b/packaging/fedora/infinito-nexus.spec @@ -1,5 +1,5 @@ Name: infinito-nexus -Version: 13.0.0 +Version: 14.0.0 Release: 1%{?dist} Summary: Meta package for Infinito.Nexus host dependencies @@ -58,6 +58,148 @@ install -d %{buildroot}%{_docdir}/%{name} %doc %{_docdir}/%{name}/DEPENDENCIES %changelog +* Sat Sep 19 2026 Kevin Veen-Birkenbach - 14.0.0-1 +- **For Users** + +- * **AI assistance inside the applications you already run.** A gateway role, +- *svc-ai-litellm*, fronts OpenAI, Anthropic and OpenRouter from declared keys, with +- *svc-ai-ollama* and the new *svc-ai-lmstudio* serving local models from one shared +- catalogue, so inference can stay inside the deployment. Sixteen applications use it for +- their own built-in AI features instead of calling a vendor directly: WordPress AI Engine, +- MediaWiki AIEditingAssistant, Moodle's AI subsystem, Zammad Smart Assist, the xWiki LLM +- extension, Discourse AI, the Matrix ChatGPT bridge, Nextcloud, Mattermost Agents, n8n, +- Open WebUI and Flowise. The gateway's admin UI signs in through Keycloak. See +- [svc-ai-litellm](roles/svc-ai-litellm/README.md) and +- [svc-ai-lmstudio](roles/svc-ai-lmstudio/README.md). + +- * **Your tools reachable from your assistants.** About twenty applications now speak MCP. +- Gitea, GitLab, Mattermost, Nextcloud, n8n, Baserow, Jenkins, Home Assistant, Moodle, +- Snipe-IT and WordPress serve their own endpoints; ten more get one through a locked-down +- sidecar. Open WebUI, Flowise and the agents discover what the deployment offers and +- register it automatically. Access is bounded rather than assumed: blocked paths are +- declared, every served path is gated, tools a provider calls mutating are withheld, +- reader and writer roles are separate, and each provider authenticates callers as its own +- service account. See [svc-ai-mcp-adapter](roles/svc-ai-mcp-adapter/README.md). + +- * **Agents as employees, with a kernel between them and the host.** *web-app-hermes* and +- *web-app-openclaw* deploy agents behind the single sign-on gate, wired to the gateway's +- models and to the MCP servers discovery hands them, each proving on deploy that it can +- actually reach them. *svc-virt-kata* runs that tier under a kernel-isolating container +- runtime, and *svc-ai-robot* embodies an agent on a dedicated device. Every sandboxed +- agent acts as its own platform account rather than borrowing yours. See +- [web-app-hermes](roles/web-app-hermes/README.md), +- [web-app-openclaw](roles/web-app-openclaw/README.md) and +- [svc-virt-kata](roles/svc-virt-kata/README.md). + +- * **Home automation joins the platform.** *web-app-homeassistant* deploys the automation +- hub, exposes it as an MCP server so agents can drive it, persists the token it mints, and +- is reachable over tor. See [web-app-homeassistant](roles/web-app-homeassistant/README.md). + +- * **Changing a password changes it everywhere.** A user declares how its password must be +- generated, the inventory pins one for every declared user, a new generator produces +- API-key shapes, the deploy rotates credentials between its two passes, and a reset command +- exists. Applications that persist an administrator password at install now realign it when +- the declared one changes: Nextcloud, Moodle, EspoCRM, Jellyfin, Listmonk, Discourse, +- Checkmk, xWiki, n8n, Matrix and the dedicated MariaDB and Postgres engines. Checkmk keeps +- a 401 fatal so a wrong password cannot lock the account out. + +- * **Onions, firewall and name resolution.** *svc-net-firewall* owns the packet filter as one +- nftables table per role. Tor carries mail to a .onion in both deploy modes, rejects forged +- sources on the guarded ports, and binds its dnsmasq listeners statically so a failed +- re-enumeration can no longer drop every one of them. The deployment declares its own +- address space once, the egress range moved out of the Docker pool, and the container +- resolver follows the live bridge instead of a guessed address. Every host-bound port +- requires an explicit onion decision. See +- [svc-net-firewall](roles/svc-net-firewall/README.md). + +- * **Application fixes you will notice.** Nextcloud installs an OnlyOffice release upstream +- actually published, keeps social login working over tor, and makes its OIDC account an +- administrator. Element shows invites again through the new room list. Moodle survives the +- container roll during installation. Mattermost keeps its configuration in Postgres instead +- of on a shared volume, and its plugin tree is per node. GitLab creates the default +- organization a headless install never gets. Jellyfin waits for health before driving the +- setup wizard. Keycloak registers redirect URIs only for the apps a play deploys, keeps the +- allowed-origins claim out of the access token, and no longer caches LDAP federation past a +- fresh grant. The platform's own health mail no longer eats the sender quota. + +- * **Operating a deployment.** New make targets reach a single deployed application: +- *compose-app-exec*, *compose-app-logs* and *compose-app-restart*. Diagnostics collect the +- logs an application writes to a file, including from busybox containers. Health probes use +- a fast start cadence with a window that outlasts a real boot, so a slow starter is no +- longer reaped. The CSP health check accepts a status code a vhost declares instead of +- dropping the vhost, and gives onion vhosts a navigation budget Tor can meet. + +- **For Developers** + +- * **MCP is declared, not wired.** A role states an *mcp* block in *meta/services.yml*, and +- the vocabulary, discovery and rendering layer derives endpoints, clients, credentials and +- network edges from it. A client-provider pairing is declared once, on the provider; +- *sys-svc-mcp-reconcile* converges clients from the complete provider set; a refused pairing +- is expressed as off rather than gated at the call site; and *svc-ai-mcp-adapter* is a +- reusable per-provider sidecar rather than a standalone application. The contract is proven +- on deploy, including refusal on *initialize* and the tool inventory itself. The design, +- its delegation rules and a machine-checkable audit live under +- [docs/contributing/design/role/services/mcp.md](docs/contributing/design/role/services/mcp.md). + +- * **Registry and variant changes.** A variant can dictate the configuration of the providers +- it pulls in, and an MCP round decides for itself what it carries, which is the breaking +- change in this release. Fifteen moving pins gained names the registry can order, every MCP +- provider is registered as a shared service, *meta/schema.yml* finished its rename to +- *meta/secrets.yml*, role scripts are sorted into per-language folders, and *meta/addons* is +- now an accepted declaration site for pins and plugins. + +- * **Lints that fail closed.** Five properties that used to pass silently now fail. The PHP +- and Ruby that roles ship are syntax-checked, and so are shell scripts that ship as Jinja +- templates. *sys-service* owns every systemd unit a role installs. Healthcheck timings are +- judged against what the base image was measured to hold, a variable-free healthcheck +- belongs in *meta/services.yml*, an unenforceable NFS state layout fails at author time, a +- sandboxed role cannot be pinned to the manager, credentials are kept out of volumes +- declared non-secret, *no_log* is required on every task carrying one and forbidden in the +- test roles, and a JavaScript default for an unset environment variable is rejected. + +- * **Pins that cannot age unnoticed.** Image pins are classified and pull references built +- from the class, a moving tag declares itself with the marker it already carries, and a +- role's images move as one release or not at all. The updater reads addon declarations, +- orders vendor patch counters and five-component versions, reads a release line written +- before the number as a family, and now also watches pinned pip requirements and Ansible +- collections. Seven roles moved off the Debian codename base onto slim, and one Ubuntu +- mirror list with failover, a short timeout and a health test serves both the package cache +- and the image builds. *js-yaml* was lifted to 4.3.2 for GHSA-2883-xcg3-v3hh. + +- * **CI selects, resumes and proves.** A fourth deploy chunk stops a short priority chunk +- stranding budget, a manual run can set the chunk size the formula would derive, and a +- retrigger can override the chunk gate. Retriggering works by role and by what never got a +- verdict, resumes behind the green window rather than the deployed one, retires suites the +- source run already passed, and cancels the run before retriggering its failed roles. Each +- role's README instructions are replayed on its smallest deploy row, in the mode its +- *services.yml* declares. Distro images build concurrently on one runner, the matrix job +- folds into the image job, and the deploy chunks no longer wait on the DNS test. + +- * **Swarm, measured rather than assumed.** The deploy waits for its own update instead of +- guessing after it, refuses to call a stack converged while it is still updating, survives a +- task that failed and came back, reports what converged rather than only what did not, and +- says why the convergence probe refused. Placement is constrained to a role's declared +- platform arch, redis and memcached sidecars get a deploy block, and a run-once service no +- longer strands the stack deploy waiting for a replica it never has. The rescue dump +- captures the contents of the state volumes. In the test lab, workers are labelled +- kata-capable only when the app is sandboxed, NFS detaches before the node containers die, +- and the sandbox tier gets a second node. + +- * **Faster gates.** Per-host setup runs once per host instead of once per application, a +- role's volumes are answered from its own file instead of a whole-tree build, the Keycloak +- RBAC root group resolves once rather than once per app, the Playwright fixture copy the +- tree copy already made is dropped, logout helpers wait for a control instead of sleeping, +- and the OIDC login helper stops waiting out probes that cannot match. Onion timeouts scale +- by tor flavor instead of a flat multiplier, and every Playwright timeout has one source. + +- * **Documentation and agent instructions.** Six new requirement documents cover the LLM +- gateway and model backends, agent employees, Home Assistant, the robot role, the MCP proxy +- expansion and native AI gateway integration. The MCP design docs are nested under their own +- folder with an audit the lints can check. The role README template asks the reader for a +- domain the deployment can resolve, enforces consistent Credits attribution, and drops the em +- dash. Documentation generators are gathered under *cli/build/docs*, and a new CLI edits role +- bonds in a matrix instead of hunting through files. + * Wed Aug 26 2026 Kevin Veen-Birkenbach - 13.0.0-1 - * **Tor as a first-class deployment axis.** The new *svc-net-tor* role routes onion - names through Tor via DNS instead of per-app proxies, offers a chutney flavor for diff --git a/pyproject.toml b/pyproject.toml index 606ff87fab..a20ef2df66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "infinito-nexus" -version = "13.0.0" +version = "14.0.0" description = "Infinito.Nexus" readme = "README.md" requires-python = ">=3.11" From b20ad3c375c2af30b912661ad337d9e1b9611f51 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sat, 19 Sep 2026 20:07:23 +0200 Subject: [PATCH 156/197] docs(web-app-openbao): the declared-user password trap is closed The note told readers to set an explicit password until a commit landed that pins one at inventory creation. That commit is an ancestor of this branch, so the workaround it prescribed is no longer needed, and the branch URL it linked answers 404 now that the branch is merged and deleted. --- roles/web-app-openbao/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-openbao/README.md b/roles/web-app-openbao/README.md index e9027bfaf1..54461abc8e 100644 --- a/roles/web-app-openbao/README.md +++ b/roles/web-app-openbao/README.md @@ -126,7 +126,7 @@ docker run --rm -it \ - `current_key_id` is derived as a short hash of the key rather than a counter. Upstream requires the identifier to match its key; a derived id cannot drift out of step with one, and no deploy has to know which generation it is. The record is written last, after both pins, so a run that dies partway leaves the previous generation intact and the next deploy still authenticates. The one uncovered window is a failure *between* the two pins โ€” the RoleID re-pinned, the SecretID not, or the reverse โ€” which leaves a pairing neither the inventory nor the state file describes. Two generations is the whole window. Anything that skips a rotation puts the store outside it, and the instance is then recoverable only from its volume backup plus the matching seal key. The concrete way to do that is `MODE_RESET`: `sys-svc-compose/tasks/01_reset.yml` removes `DIR_COMPOSITIONS` wholesale while the named volumes survive, so the raft store outlives the record of the key that wraps it. Purge the entity's volumes too when you reset โ€” the same asymmetry breaks every entity that keeps a credential inside its volume. - **Other entities still desynchronise the same way, and those are not defects.** A surviving `mariadb_data` fails its healthcheck with `Access denied for user 'root'@'127.0.0.1'` for exactly the reason above, without a rotation path of its own. CI never sees it because every run starts clean; locally, run `make compose-entity-purge apps=` before redeploying. -- **A role-declared account cannot be logged in as unless it sets a password.** The tiers were first covered by two role-local accounts, `baooperator` and `baoreader`, which deployed and joined their RBAC groups correctly but returned 403 on every LDAP bind: a user declared without an explicit `password` inherits the `strong_password` fallback expression, which is re-evaluated on every render (`secrets.choice`, nothing persisted), so the value `svc-db-openldap` writes with `ldap_passwd` and the value `playwright.env.j2` renders can be different strings. Both accounts are gone now โ€” the tiers are covered by moving `biber` through the groups instead โ€” but the trap applies to any future role-declared account something has to authenticate as. Until [`feature/ai-agent-employees`](https://github.com/kevinveenbirkenbach/infinito-nexus-core/tree/refs/heads/feature/ai-agent-employees) lands (`f4398e1c5` pins a password per declared user at inventory creation), such an account needs an explicit `password:`, as `user-administrator` and `web-app-semaphore` do. +- **A role-declared account cannot be logged in as unless it sets a password.** The tiers were first covered by two role-local accounts, `baooperator` and `baoreader`, which deployed and joined their RBAC groups correctly but returned 403 on every LDAP bind: a user declared without an explicit `password` inherits the `strong_password` fallback expression, which is re-evaluated on every render (`secrets.choice`, nothing persisted), so the value `svc-db-openldap` writes with `ldap_passwd` and the value `playwright.env.j2` renders can be different strings. Both accounts are gone now โ€” the tiers are covered by moving `biber` through the groups instead. [`f4398e1c5`](https://github.com/infinito-nexus/core/commit/f4398e1c58f3ce7d8d429472b2a023033242d747) closed the trap at the source: inventory creation now decides the value once, after the roles are resolved, so a declared user no longer carries a template that re-renders per use site. An explicit `password:`, as `user-administrator` and `web-app-semaphore` set, is therefore no longer required for a role-declared account to be authenticated as. - **Alert rules ship with the role.** `templates/prometheus/alert_rules.yml.j2` is discovered by the `alert_rule_roles` lookup and concatenated into the Prometheus rule file, so the seal alert lives next to the thing it watches. The fragment must start at the `groups:` item level (two-space-indented `- name:` entries). ### Backup and restore From 2b51f1cb62e59073ba278e06ea16083b4851e69e Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sat, 19 Sep 2026 20:58:05 +0200 Subject: [PATCH 157/197] fix(openbao): pin the seal key out of the per-deploy rotation The static seal carries one generation. Rotating the key on every deploy spends that slack continuously, so a run that skips a generation, or dies between the two AppRole pins, leaves the raft store readable only from a volume backup plus the key that wrapped it -- and the recovery keys that would authorise a rekey are discarded at init. Frequent rotation buys little here in return. The key lives in the vault-encrypted inventory either way, and whoever holds that inventory holds the AppRole too, so they reach the data without needing the seal key. An operator now rotates it deliberately by changing the inventory value; the previous_key path stays in place to carry that. The AppRole keeps rotating per deploy. It is the live authentication path, and the fallback login exists to absorb exactly that. --- roles/web-app-openbao/meta/secrets.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/web-app-openbao/meta/secrets.yml b/roles/web-app-openbao/meta/secrets.yml index 8bc1ec482a..bcf1f917fd 100644 --- a/roles/web-app-openbao/meta/secrets.yml +++ b/roles/web-app-openbao/meta/secrets.yml @@ -2,9 +2,12 @@ credentials: seal_key: description: 32-byte AES-256-GCM-96 key for the OpenBao static seal, injected as env://OPENBAO_SEAL_KEY. Every backup of the raft volume is undecryptable - without it; its authoritative home is the encrypted inventory. + without it; its authoritative home is the encrypted inventory. Pinned because the static seal carries only one generation, so a deploy that skips a + generation, or dies between the two AppRole pins, puts the store outside that window and leaves it recoverable only from a volume backup plus the key + that wraps it. Rotate it deliberately by changing the inventory value; openbao.hcl.j2 still renders the previous_key pair for that. algorithm: random_hex_32 validation: non_empty_string + rotatable: false approle_role_id: description: RoleID of the Ansible AppRole, written to auth/approle/role/ansible/role-id at bootstrap so later deploys authenticate without a stored root token. From 56c3da4a146da71b9abe6eb1cbbdc6e2ef8c9010 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sat, 19 Sep 2026 20:58:14 +0200 Subject: [PATCH 158/197] test(openbao): pin which credentials rotate Nothing stopped the seal key drifting back into the per-deploy rotation, and the regression is invisible until a deploy is skipped and the node cannot unseal. Assert the pin on the seal key and its absence on both AppRole credentials. --- .../test_credential_rotation.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py b/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py index 3af7af87f4..554b6c0d26 100644 --- a/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py +++ b/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py @@ -1,8 +1,9 @@ """Guards for the per-deploy credential rotation path of web-app-openbao. The platform regenerates generated-algorithm credentials on every deploy, so the -seal key and AppRole a run is handed differ from the ones the running instance -was configured with. Three things carry that transition, and each fails silently +AppRole a run is handed differs from the one the running instance was configured +with. The seal key is pinned out of that and moves only when an operator changes +it, but the same transition then has to carry it. Each part fails silently rather than loudly when broken -- a sealed node and an unusable AppRole look identical to ordinary drift. These tests pin them. """ @@ -15,6 +16,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined from utils.cache.files import read_text from utils.cache.yaml import load_yaml_any +from utils.roles.mapping import ROLE_FILE_META_SECRETS from . import PROJECT_ROOT @@ -53,6 +55,32 @@ def tasks_of(name: str) -> list[dict]: return load_yaml_any(str(TASKS_DIR / name)) +class TestWhichCredentialsRotate(unittest.TestCase): + """The seal key survives a deploy; the AppRole does not. + + Rotating the seal key on every deploy spends the static seal's single + generation of slack for nothing. A run that skips a generation, or dies + between the two AppRole pins, leaves the raft store readable only from a + volume backup plus the key that wrapped it, and the role keeps no recovery + keys to fall back on. The AppRole is the opposite case: it is the live + authentication path, and absorbing its rotation is what the fallback login + in 01_init.yml exists for. + """ + + def setUp(self): + self.credentials = load_yaml_any(str(ROLE_DIR / ROLE_FILE_META_SECRETS))[ + "credentials" + ] + + def test_the_seal_key_is_pinned(self): + self.assertIs(self.credentials["seal_key"].get("rotatable"), False) + + def test_both_approle_credentials_still_rotate(self): + for name in ("approle_role_id", "approle_secret_id"): + with self.subTest(credential=name): + self.assertIsNot(self.credentials[name].get("rotatable"), False) + + class TestSealStanzaRotation(unittest.TestCase): def _render(self, rotating: bool) -> str: env = jinja_env(ROLE_DIR / "templates") From fcfff81f640e764da1459a86078d107c8bfe8bd3 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sat, 19 Sep 2026 20:58:25 +0200 Subject: [PATCH 159/197] docs(openbao): correct what the rotation record covers The header claimed every generated credential rotates per deploy. The seal key no longer does; it is recorded for the deploy that follows an operator changing it. --- roles/web-app-openbao/tasks/06_rotate.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/roles/web-app-openbao/tasks/06_rotate.yml b/roles/web-app-openbao/tasks/06_rotate.yml index aa8fd31be8..f7ed0ba83a 100644 --- a/roles/web-app-openbao/tasks/06_rotate.yml +++ b/roles/web-app-openbao/tasks/06_rotate.yml @@ -1,8 +1,10 @@ --- # Re-pin the AppRole to the credentials this deploy was handed, then record what -# was applied. The platform rotates every generated credential per deploy, so the -# values the next deploy receives will differ from these; the state file is what -# lets that deploy still authenticate before it re-pins in turn. +# was applied. The platform rotates the AppRole per deploy, so the values the +# next deploy receives will differ from these; the state file is what lets that +# deploy still authenticate before it re-pins in turn. The seal key is pinned +# and normally repeats, and is recorded for the deploy that follows an operator +# changing it. - name: "๐Ÿ”‘ Re-pin the RoleID to the value this deploy was handed" ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node container exec {{ OPENBAO_LOCAL_CID | quote }} From 48333bb50a5637b3d3e362c9409141647e69d910 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sat, 19 Sep 2026 20:59:50 +0200 Subject: [PATCH 160/197] docs(openbao): restate why the key id is derived The rationale rested on the key rotating every deploy, which it no longer does. Deriving the id from the key is what keeps it in step with the key, however rarely that happens. --- roles/web-app-openbao/vars/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/web-app-openbao/vars/main.yml b/roles/web-app-openbao/vars/main.yml index d5caa506b1..0de0720fe1 100644 --- a/roles/web-app-openbao/vars/main.yml +++ b/roles/web-app-openbao/vars/main.yml @@ -20,9 +20,9 @@ OPENBAO_SEAL_KEY: "{{ lookup('config', application_id, 'secrets.credentials.seal OPENBAO_APPROLE_ROLE_ID: "{{ lookup('config', application_id, 'secrets.credentials.approle_role_id') }}" OPENBAO_APPROLE_SECRET_ID: "{{ lookup('config', application_id, 'secrets.credentials.approle_secret_id') }}" -# Rationale: the platform rotates every generated credential on each deploy, so -# the key id is derived from the key instead of counted โ€” upstream requires the -# identifier to match its key, and a derived id cannot drift out of step with one. +# Rationale: the key id is derived from the key instead of counted โ€” upstream +# requires the identifier to match its key, and a derived id cannot drift out of +# step with one however rarely an operator rotates the key. OPENBAO_SEAL_KEY_ID: "{{ (OPENBAO_SEAL_KEY | hash('sha256'))[:16] }}" # Rationale: what the previous deploy applied, read back before the config is From 9248d31e025d2d47f137ee57e4a62e674390c578 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sat, 19 Sep 2026 21:00:33 +0200 Subject: [PATCH 161/197] docs(openbao): document the pinned seal key Records why the key is held out of the rotation, how to rotate it deliberately, and what pinning costs. Also corrects the MODE_RESET warning: with the seal key pinned the store stays readable, and what a reset destroys is the AppRole record, leaving no administrative path back in. --- roles/web-app-openbao/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/web-app-openbao/README.md b/roles/web-app-openbao/README.md index 54461abc8e..a6be358039 100644 --- a/roles/web-app-openbao/README.md +++ b/roles/web-app-openbao/README.md @@ -107,6 +107,7 @@ docker run --rm -it \ ## Developer Notes - **The seal key is the whole ballgame.** The `static` seal encrypts the raft root key with a 32-byte key generated into the ansible-vault-encrypted inventory as `secrets.credentials.seal_key` and injected as `env://OPENBAO_SEAL_KEY`. A backup of the raft volume is **undecryptable without that key**, and the key is *not* covered by `svc-bkp-secrets-2-local` (whose sources are `DIR_SECRETS`, the root CA and the ACME material). The inventory is its authoritative home โ€” back the inventory up accordingly. Upstream considers a static seal appropriate only where an existing source of trust already holds the key; here that source is the encrypted inventory, at the same trust level as every other platform credential. +- **The seal key is pinned, and that is deliberate.** `meta/secrets.yml` marks it `rotatable: false`, so the per-deploy rotation skips it. The static seal carries one generation of slack, and rotating on every deploy spends all of it continuously: a run that skips a generation, or dies between the two AppRole pins, leaves the raft store readable only from a volume backup plus the key that wrapped it โ€” and the recovery keys that would otherwise authorise a rekey were discarded at init. The key gains little from frequent rotation in exchange, because it lives in the vault-encrypted inventory either way, and an attacker holding that inventory holds the AppRole too and does not need the seal key to reach the data. Rotate it deliberately instead, by changing `secrets.credentials.seal_key` in the inventory: `templates/openbao.hcl.j2` then renders the `previous_key_id`/`previous_key` pair, the node unseals with the old key and rewraps to the new one, and the next deploy sees them equal again. The cost of pinning is that a leaked inventory backup stays valid against future volume snapshots instead of ageing out in two deploys. - **Recovery keys are generated and then deliberately discarded.** Initialising with an auto-seal yields *recovery* keys rather than unseal shares. They cannot decrypt the root key โ€” only the seal key can โ€” so they are not disaster-recovery material for the data; they only authorise `operator generate-root` and `operator rekey`. Persisting them would mean writing OpenBao-generated secrets back into the inventory, which is exactly what the AppRole bootstrap exists to avoid, so the play does not keep them. **The consequence is explicit: the Ansible AppRole is the only administrative path into this instance.** If its RoleID/SecretID are changed out-of-band inside OpenBao, there is no way back in and the instance must be rebuilt from the volume backup plus the seal key. Re-running the role does not repair that, because reconciliation itself authenticates with the AppRole โ€” the rotation fallback below only covers the credentials the role itself applied. - **Bootstrap leaves no stored root token.** The first deploy runs `bao operator init`, holds the root token in memory only, enables AppRole, pins both the RoleID and the SecretID to inventory-generated values via `auth/approle/role//role-id` and `.../custom-secret-id`, then revokes the root token. Every later deploy logs in with those inventory credentials, so nothing OpenBao generates is ever written back into the inventory. Tokens reach the CLI through stdin and the container token helper, never through process arguments, and the helper file is removed at the end of the run. - **Policies live outside the config directory.** OpenBao parses every file in `-config` as server configuration, so the RBAC policy `.hcl` files are mounted at `/openbao/policies/` instead of `/openbao/config/`. @@ -117,14 +118,14 @@ docker run --rm -it \ - **Do not alert on `vault_core_unsealed`.** OpenBao keeps Vault's `vault_*` metric namespace, and on a healthy unsealed node this gauge is emitted as `vault_core_unsealed{cluster=""} 0` โ€” an alert on `== 0` fires permanently. Use `/v1/sys/health` (503 while sealed) as the seal signal; `vault_core_active{cluster="โ€ฆ"} == 1` corroborates from the metrics side. - **Swarm:** single replica, `placement: manager`, `nfs: false` on the raft volume and `service_update_order = stop-first` โ€” raft's BoltDB store is mmap + `fsync` based, so it must stay node-local and must never have two tasks writing the same directory. - **The RBAC tiers are proven by moving `biber` through the groups, not by dedicated accounts.** `files/playwright/test-rbac-groups.js` signs the administrator into LAM, adds `biber` to `cn=web-app-openbao-,ou=roles` through LAM's Multi edit tool, logs `biber` into OpenBao over LDAP and asserts both the policy set and the capability that tier grants, then removes him again and asserts the revoke. One account covers all three tiers, and the grant and the revoke are observed rather than assumed โ€” a permanently-assigned member can only ever show a steady state. LAM's *group list* cannot be used for this: it renders `posixGroup` entries, while the RBAC role groups are `groupOfNames` under `ou=roles`. LAM applies the change asynchronously, so the spec polls the resulting policy set instead of LAM's own progress wording. -- **Every deploy rotates the credentials, and the role is built around that.** The platform regenerates generated-algorithm credentials on each deploy โ€” that is how a changed Ansible password reaches the application instead of silently diverging from it. For OpenBao it means the `seal_key` and the AppRole `role_id`/`secret_id` a run is handed are *not* the ones the running instance was configured with, which would otherwise leave the node sealed (`initialized: true, sealed: true`) and the AppRole answering `invalid role or secret ID`. The role therefore carries the transition itself: +- **Every deploy rotates the AppRole, and the role is built around that.** The platform regenerates generated-algorithm credentials on each deploy โ€” that is how a changed Ansible password reaches the application instead of silently diverging from it. For OpenBao it means the `role_id`/`secret_id` a run is handed are *not* the ones the running instance was configured with, which would otherwise leave the AppRole answering `invalid role or secret ID`. The role therefore carries the transition itself: - `tasks/06_rotate.yml` records what each deploy actually applied in `.applied.json` inside the compose project directory, `0600`, root-owned. The directory already holds the seal key in plaintext in `.env/env`, so this adds no new exposure class. - `tasks/00_core.yml` reads that file back **before** `sys-stk-backend` renders `env.j2` โ€” the rendered env must already carry `OPENBAO_SEAL_KEY_PREVIOUS`, or the node comes up with no way to decrypt its own store. - `templates/openbao.hcl.j2` emits the `previous_key_id`/`previous_key` pair only while the two keys differ. The static seal supports `n-1` โ†’ `n` rotation, so the node unseals with the old key and rewraps to the new one. - `tasks/01_init.yml` attempts the AppRole login with the inventory credentials, falls back to the recorded ones, and fails loudly when neither is accepted. Whichever one was accepted is the SecretID OpenBao actually holds, so that is what `tasks/06_rotate.yml` destroys after registering the new one โ€” rotation that only adds would revoke nothing. - `auth/approle/role//custom-secret-id` is **not** idempotent: writing a SecretID the role already holds answers `500 โ€ฆ SecretID is already registered`. The swap is therefore gated on the value having actually changed, which is also why the bootstrap in `tasks/02_auth.yml` pins directly instead of sharing the rotation path โ€” at that point nothing is registered yet. - `current_key_id` is derived as a short hash of the key rather than a counter. Upstream requires the identifier to match its key; a derived id cannot drift out of step with one, and no deploy has to know which generation it is. - The record is written last, after both pins, so a run that dies partway leaves the previous generation intact and the next deploy still authenticates. The one uncovered window is a failure *between* the two pins โ€” the RoleID re-pinned, the SecretID not, or the reverse โ€” which leaves a pairing neither the inventory nor the state file describes. Two generations is the whole window. Anything that skips a rotation puts the store outside it, and the instance is then recoverable only from its volume backup plus the matching seal key. The concrete way to do that is `MODE_RESET`: `sys-svc-compose/tasks/01_reset.yml` removes `DIR_COMPOSITIONS` wholesale while the named volumes survive, so the raft store outlives the record of the key that wraps it. Purge the entity's volumes too when you reset โ€” the same asymmetry breaks every entity that keeps a credential inside its volume. + The record is written last, after both pins, so a run that dies partway leaves the previous generation intact and the next deploy still authenticates. The one uncovered window is a failure *between* the two pins โ€” the RoleID re-pinned, the SecretID not, or the reverse โ€” which leaves a pairing neither the inventory nor the state file describes, and no recovery keys are kept to get back in. Two generations is the whole window for the AppRole. Discarding the state file while the volume survives puts the instance outside it: OpenBao still holds the previous SecretID, the inventory has already rotated past it, and the fallback has nothing left to read. `MODE_RESET` does exactly that โ€” `sys-svc-compose/tasks/01_reset.yml` removes `DIR_COMPOSITIONS` wholesale while the named volumes persist. The store itself stays readable, because the seal key is pinned and still comes from the inventory, but no administrative path into it remains. Purge the entity's volumes too when you reset โ€” the same asymmetry breaks every entity that keeps a credential inside its volume. - **Other entities still desynchronise the same way, and those are not defects.** A surviving `mariadb_data` fails its healthcheck with `Access denied for user 'root'@'127.0.0.1'` for exactly the reason above, without a rotation path of its own. CI never sees it because every run starts clean; locally, run `make compose-entity-purge apps=` before redeploying. - **A role-declared account cannot be logged in as unless it sets a password.** The tiers were first covered by two role-local accounts, `baooperator` and `baoreader`, which deployed and joined their RBAC groups correctly but returned 403 on every LDAP bind: a user declared without an explicit `password` inherits the `strong_password` fallback expression, which is re-evaluated on every render (`secrets.choice`, nothing persisted), so the value `svc-db-openldap` writes with `ldap_passwd` and the value `playwright.env.j2` renders can be different strings. Both accounts are gone now โ€” the tiers are covered by moving `biber` through the groups instead. [`f4398e1c5`](https://github.com/infinito-nexus/core/commit/f4398e1c58f3ce7d8d429472b2a023033242d747) closed the trap at the source: inventory creation now decides the value once, after the roles are resolved, so a declared user no longer carries a template that re-renders per use site. An explicit `password:`, as `user-administrator` and `web-app-semaphore` set, is therefore no longer required for a role-declared account to be authenticated as. - **Alert rules ship with the role.** `templates/prometheus/alert_rules.yml.j2` is discovered by the `alert_rule_roles` lookup and concatenated into the Prometheus rule file, so the seal alert lives next to the thing it watches. The fragment must start at the `groups:` item level (two-space-indented `- name:` entries). From 7affa3d163c69031235b18c2a31268339bfab88c Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sun, 20 Sep 2026 01:08:56 +0200 Subject: [PATCH 162/197] feat(openbao): add a break-glass path through the recovery key Mints a root token with bao operator generate-root and hands over to 02_auth.yml, which re-pins both AppRole halves to the inventory values. The four protocol steps run as separate tasks because the token comes back XOR-wrapped with a one-time password and the image carries neither python nor jq, so the CLI performs each step and Ansible parses the JSON -- the same split the rest of the role already uses for bao status. --- roles/web-app-openbao/tasks/utils/recover.yml | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 roles/web-app-openbao/tasks/utils/recover.yml diff --git a/roles/web-app-openbao/tasks/utils/recover.yml b/roles/web-app-openbao/tasks/utils/recover.yml new file mode 100644 index 0000000000..1b3e0ceab5 --- /dev/null +++ b/roles/web-app-openbao/tasks/utils/recover.yml @@ -0,0 +1,84 @@ +--- +# Break-glass: mint a root token from the recovery key and rebuild the AppRole +# from the inventory. Reached only when neither the credentials this deploy was +# handed nor the ones the previous deploy recorded are accepted, which without +# this path ends the run with no administrative way into a node whose data is +# still perfectly readable. +# +# The protocol is four steps and the token it yields is XOR-wrapped with a +# one-time password, so each step runs through the bao CLI and Ansible parses +# the result -- the image carries no python or jq to do it in-container. +# +# The last step hands over to 02_auth.yml, which re-pins both halves to the +# inventory values and revokes the root token it was given. The AppRole this +# deploy holds is therefore live on return, and 06_rotate.yml finds nothing +# left to swap. + +# Exception: an attempt left behind by an interrupted run makes -init fail with +# "root generation already in progress". +- name: "๐Ÿงน Cancel any root generation left in progress" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec {{ OPENBAO_LOCAL_CID | quote }} + bao operator generate-root -cancel + changed_when: false + failed_when: false + delegate_to: "{{ OPENBAO_HOST_NODE }}" + +- name: "๐ŸŽฒ Generate the one-time password the root token is wrapped with" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec {{ OPENBAO_LOCAL_CID | quote }} + bao operator generate-root -generate-otp + register: openbao_root_otp + changed_when: false + delegate_to: "{{ OPENBAO_HOST_NODE }}" + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "๐Ÿš‘ Begin root generation" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec {{ OPENBAO_LOCAL_CID | quote }} + bao operator generate-root -format=json + -init -otp={{ openbao_root_otp.stdout | trim | quote }} + register: openbao_root_attempt + changed_when: true + delegate_to: "{{ OPENBAO_HOST_NODE }}" + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "๐Ÿ”‘ Authorise the attempt with the recovery key" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec -i {{ OPENBAO_LOCAL_CID | quote }} + bao operator generate-root -format=json + -nonce={{ (openbao_root_attempt.stdout | from_json).nonce | quote }} - + args: + stdin: "{{ OPENBAO_RECOVERY_KEY }}" + stdin_add_newline: false + register: openbao_root_update + changed_when: true + delegate_to: "{{ OPENBAO_HOST_NODE }}" + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "๐Ÿ›‘ Fail when the recovery key did not complete root generation" + ansible.builtin.fail: + msg: >- + The recovery key held for {{ application_id }} was rejected. Neither + AppRole is accepted either, so this instance has no administrative path + left and must be rebuilt from its volume backup plus the seal key. + when: not ((openbao_root_update.stdout | from_json).complete | default(false) | bool) + +- name: "๐Ÿ”“ Unwrap the generated root token" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec {{ OPENBAO_LOCAL_CID | quote }} + bao operator generate-root + -decode={{ (openbao_root_update.stdout | from_json).encoded_token | quote }} + -otp={{ openbao_root_otp.stdout | trim | quote }} + register: openbao_root_decoded + changed_when: false + delegate_to: "{{ OPENBAO_HOST_NODE }}" + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "๐Ÿ”ง Hold the generated root token in memory only" + ansible.builtin.set_fact: + openbao_root_token: "{{ openbao_root_decoded.stdout | trim }}" + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "๐Ÿ”‘ Rebuild the Ansible AppRole from the inventory credentials" + ansible.builtin.include_tasks: ../02_auth.yml From b129189f50ace5dba132eabf3fb23a32a62d8fd5 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sun, 20 Sep 2026 01:22:34 +0200 Subject: [PATCH 163/197] feat(openbao): resolve the recovery key from the platform token store DIR_SECRETS is the durable home a role actually has. The inventory would be the natural one, but no role writes host_vars -- the CLI writes it before the play. DIR_SECRETS outlives MODE_RESET and a replaced swarm manager, both of which take .applied.json with them, and svc-bkp-secrets-2-local backs it up. --- roles/web-app-openbao/vars/main.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/roles/web-app-openbao/vars/main.yml b/roles/web-app-openbao/vars/main.yml index 0de0720fe1..1564c6e4bc 100644 --- a/roles/web-app-openbao/vars/main.yml +++ b/roles/web-app-openbao/vars/main.yml @@ -31,6 +31,19 @@ OPENBAO_SEAL_KEY_ID: "{{ (OPENBAO_SEAL_KEY | hash('sha256'))[:16] }}" OPENBAO_APPLIED_STATE_HOST: "{{ [lookup('container', application_id, 'directories.instance'), '.applied.json'] | path_join }}" OPENBAO_APPLIED: "{{ openbao_applied_state | default({}, true) }}" OPENBAO_SEAL_KEY_PREVIOUS: "{{ OPENBAO_APPLIED.seal_key | default('', true) }}" + +# Rationale: the recovery key comes from OpenBao at init rather than from the +# inventory generator, and sys-token-store is where the platform keeps exactly +# that class of secret. DIR_SECRETS outlives both MODE_RESET and a manager +# replacement, each of which takes .applied.json with it, and +# svc-bkp-secrets-2-local already covers it. +OPENBAO_RECOVERY_USER_KEY: "administrator" +OPENBAO_RECOVERY_TOKEN_APP: "{{ application_id }}-recovery" +OPENBAO_RECOVERY_KEY: >- + {{ (sys_token_store_users | default({}, true)) + .get(OPENBAO_RECOVERY_USER_KEY, {}) + .get('tokens', {}) + .get(OPENBAO_RECOVERY_TOKEN_APP, '') }} OPENBAO_SEAL_KEY_PREVIOUS_ID: "{{ (OPENBAO_SEAL_KEY_PREVIOUS | hash('sha256'))[:16] if OPENBAO_SEAL_KEY_PREVIOUS else '' }}" OPENBAO_SEAL_ROTATING: "{{ (OPENBAO_SEAL_KEY_PREVIOUS | length > 0) and (OPENBAO_SEAL_KEY_PREVIOUS != OPENBAO_SEAL_KEY) }}" From f1c217b685f1186a15c12d78d847d6db98e521c9 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sun, 20 Sep 2026 01:22:46 +0200 Subject: [PATCH 164/197] feat(openbao): keep the recovery key and fall back to it bao operator init issued a recovery key on the first deploy and the play read only root_token from the JSON, dropping it. That left the AppRole as the sole administrative path, so losing both halves of it ended the deploy for good on an instance whose data was still readable. The key is now persisted at init and loaded before the logins that may need it. When neither the inventory AppRole nor the recorded one is accepted, the run rebuilds the AppRole from it instead of failing. The hard fail remains for the case where there is no key either. The three first-deploy tasks move into a block: a third sibling sharing the same when is what test_no_repeated_when_runs rejects. --- roles/web-app-openbao/tasks/01_init.yml | 93 +++++++++++++++++++------ 1 file changed, 73 insertions(+), 20 deletions(-) diff --git a/roles/web-app-openbao/tasks/01_init.yml b/roles/web-app-openbao/tasks/01_init.yml index b22bf3ecb5..93a405d416 100644 --- a/roles/web-app-openbao/tasks/01_init.yml +++ b/roles/web-app-openbao/tasks/01_init.yml @@ -25,22 +25,39 @@ if (openbao_status_raw.stdout | default('') | trim) | length > 0 else false }} -- name: "๐Ÿ” Initialise OpenBao (first deploy only)" - ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node - container exec {{ OPENBAO_LOCAL_CID | quote }} - bao operator init -format=json -recovery-shares=1 -recovery-threshold=1 - register: openbao_init_raw - changed_when: openbao_init_raw.rc == 0 - delegate_to: "{{ OPENBAO_HOST_NODE }}" - no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" +- name: "๐Ÿ” Initialise OpenBao and keep what only the first deploy ever sees" when: not (openbao_initialized | bool) + block: + - name: "๐Ÿ” Initialise OpenBao (first deploy only)" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec {{ OPENBAO_LOCAL_CID | quote }} + bao operator init -format=json -recovery-shares=1 -recovery-threshold=1 + register: openbao_init_raw + changed_when: openbao_init_raw.rc == 0 + delegate_to: "{{ OPENBAO_HOST_NODE }}" + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" -- name: "๐Ÿ”ง Hold the bootstrap root token in memory only" - ansible.builtin.set_fact: - openbao_root_token: >- - {{ (openbao_init_raw.stdout | from_json).root_token }} - no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" - when: not (openbao_initialized | bool) + - name: "๐Ÿ”ง Hold the bootstrap root token in memory only" + ansible.builtin.set_fact: + openbao_root_token: >- + {{ (openbao_init_raw.stdout | from_json).root_token }} + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + + # Exception: the only authority here that never rotates. Without it the + # AppRole is the sole administrative path, so losing the recorded pair -- + # a reset, a replaced manager, a run that dies between the two pins in + # 06_rotate.yml -- locks the instance out for good while its data stays + # perfectly readable. + - name: "๐Ÿ’พ Persist the recovery key through sys-token-store" + ansible.builtin.include_role: + name: sys-token-store + tasks_from: write.yml + vars: + sys_token_store_user_key: "{{ OPENBAO_RECOVERY_USER_KEY }}" + sys_token_store_app: "{{ OPENBAO_RECOVERY_TOKEN_APP }}" + sys_token_store_token: >- + {{ (openbao_init_raw.stdout | from_json).recovery_keys_b64 | first }} + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" - name: "โณ Wait until the static seal has auto-unsealed the node" ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node @@ -59,6 +76,12 @@ include_tasks: 02_auth.yml when: not (openbao_initialized | bool) +- name: "๐Ÿงฉ Load the recovery key from the platform token store" + ansible.builtin.include_role: + name: sys-token-store + tasks_from: 01_load.yml + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + - name: "๐Ÿ” Log in with the Ansible AppRole" ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node container exec -i {{ OPENBAO_LOCAL_CID | quote }} @@ -93,24 +116,54 @@ - openbao_approle_login.rc != 0 - OPENBAO_APPLIED.approle_role_id is defined +# Rationale: the AppRole is the only administrative path into the instance, so +# losing both halves of it used to end the deploy permanently. The recovery key +# does not rotate, which is what makes it a way back. +- name: "๐Ÿš‘ Rebuild the AppRole from the recovery key when neither login is accepted" + when: + - openbao_approle_login.rc != 0 + - (openbao_approle_login_previous.rc | default(1)) != 0 + - (OPENBAO_RECOVERY_KEY | trim) | length > 0 + block: + - name: "๐Ÿš‘ Mint a root token and re-pin the AppRole" + ansible.builtin.include_tasks: utils/recover.yml + + - name: "๐Ÿ” Log in with the rebuilt AppRole" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec -i {{ OPENBAO_LOCAL_CID | quote }} + bao write -field=token auth/approle/login + role_id={{ OPENBAO_APPROLE_ROLE_ID | quote }} secret_id=- + args: + stdin: "{{ OPENBAO_APPROLE_SECRET_ID }}" + stdin_add_newline: false + register: openbao_approle_login_recovered + changed_when: false + failed_when: false + delegate_to: "{{ OPENBAO_HOST_NODE }}" + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + - name: "๐Ÿ”ง Keep whichever AppRole login succeeded" ansible.builtin.set_fact: openbao_approle_token: >- {{ openbao_approle_login.stdout if openbao_approle_login.rc == 0 - else openbao_approle_login_previous.stdout | default('') }} + else (openbao_approle_login_previous.stdout + if (openbao_approle_login_previous.rc | default(1)) == 0 + else openbao_approle_login_recovered.stdout | default('')) }} openbao_secret_id_registered: >- {{ OPENBAO_APPROLE_SECRET_ID - if openbao_approle_login.rc == 0 + if (openbao_approle_login.rc == 0 + or (openbao_approle_login_recovered.rc | default(1)) == 0) else OPENBAO_APPLIED.approle_secret_id | default('') }} no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" -- name: "๐Ÿ›‘ Fail when neither the current nor the previously applied AppRole is accepted" +- name: "๐Ÿ›‘ Fail when no AppRole and no recovery key are accepted" ansible.builtin.fail: msg: >- - Neither the AppRole credentials from the inventory nor the ones recorded in - {{ OPENBAO_APPLIED_STATE_HOST }} were accepted. The instance can only be - recovered from its volume backup plus the matching seal key. + Neither the AppRole credentials from the inventory, nor the ones recorded + in {{ OPENBAO_APPLIED_STATE_HOST }}, nor a recovery key in the platform + token store were accepted. The instance can only be recovered from its + volume backup plus the matching seal key. when: (openbao_approle_token | trim) | length == 0 - name: "๐Ÿ”ง Store the AppRole token in the container token helper" From 19fddfddaaa843d8d7f221bdf622a8d4c8cb5feb Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sun, 20 Sep 2026 01:23:02 +0200 Subject: [PATCH 165/197] test(openbao): pin the recovery key and the path it opens Six assertions: the key is persisted on the deploy that creates it, it is loaded before the logins that may need it, recovery is gated on both logins having failed and a key existing, the rebuilt AppRole is the inventory one rather than the recorded one, the recovery path ends by re-pinning, and a rejected key fails loudly. Recovering to the recorded pair instead of the inventory pair would strand the next deploy the same way, which is why that one is asserted directly. --- .../test_credential_rotation.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py b/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py index 554b6c0d26..164b2a90cd 100644 --- a/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py +++ b/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py @@ -251,6 +251,98 @@ class TestTheSecretIdSwapIsGuarded(unittest.TestCase): self.assertNotIn("when", task) +class TestTheRecoveryKeyIsKeptAndUsable(unittest.TestCase): + """The one authority that does not rotate, and the way back it buys. + + `bao operator init` issues a recovery key once and nothing else ever + reissues it. Dropping it leaves the AppRole as the only administrative + path, so a reset, a replaced manager, or a run that dies between the two + pins in 06_rotate.yml locks the instance out for good while its data stays + readable. It goes to sys-token-store rather than the inventory: no role + writes host_vars, and DIR_SECRETS outlives both DIR_COMPOSITIONS and the + node, and is covered by svc-bkp-secrets-2-local. + """ + + def setUp(self): + self.tasks = tasks_of("01_init.yml") + + def _init_block(self) -> list[dict]: + return next( + task["block"] + for task in self.tasks + if "block" in task and "openbao_initialized" in str(task.get("when", "")) + ) + + def test_the_key_is_persisted_on_the_deploy_that_creates_it(self): + store = next( + task + for task in self._init_block() + if task.get("ansible.builtin.include_role", {}).get("name") + == "sys-token-store" + ) + self.assertEqual( + store["ansible.builtin.include_role"]["tasks_from"], "write.yml" + ) + self.assertIn("recovery_keys_b64", store["vars"]["sys_token_store_token"]) + + def test_the_key_is_loaded_before_the_logins_that_may_need_it(self): + load = next( + i + for i, task in enumerate(self.tasks) + if task.get("ansible.builtin.include_role", {}).get("tasks_from") + == "01_load.yml" + ) + login = next( + i + for i, task in enumerate(self.tasks) + if task.get("register") == "openbao_approle_login" + ) + self.assertLess(load, login) + + def test_recovery_runs_only_when_both_logins_failed_and_a_key_exists(self): + block = next( + task + for task in self.tasks + if "openbao_approle_login_recovered" in str(task) + ) + conditions = " ".join(block["when"]) + self.assertIn("openbao_approle_login.rc != 0", conditions) + self.assertIn("openbao_approle_login_previous.rc", conditions) + self.assertIn("OPENBAO_RECOVERY_KEY", conditions) + + def test_the_rebuilt_approle_is_the_one_the_inventory_holds(self): + """Recovering to the recorded pair would strand the deploy again. + + 02_auth.yml re-pins both halves to the inventory values, so the login + that proves recovery worked has to use those, not the applied ones. + """ + block = next( + task + for task in self.tasks + if "openbao_approle_login_recovered" in str(task) + ) + login = next( + task + for task in block["block"] + if task.get("register") == "openbao_approle_login_recovered" + ) + self.assertIn("OPENBAO_APPROLE_ROLE_ID", login["ansible.builtin.shell"]) + self.assertIn("OPENBAO_APPROLE_SECRET_ID", login["args"]["stdin"]) + + def test_the_recovery_path_ends_by_rebuilding_the_approle(self): + tasks = tasks_of("utils/recover.yml") + self.assertEqual( + tasks[-1]["ansible.builtin.include_tasks"], + "../02_auth.yml", + "minting a root token without re-pinning leaves the deploy no better off", + ) + + def test_a_rejected_recovery_key_fails_loudly(self): + tasks = tasks_of("utils/recover.yml") + guard = next(task for task in tasks if "ansible.builtin.fail" in task) + self.assertIn("complete", guard["when"]) + + class TestTheRegisteredSecretIdComesFromTheLogin(unittest.TestCase): def test_it_is_the_credential_that_was_just_accepted(self): """Deriving it from the state file instead would be wrong. From 67ac5a3c59d67f9cb0d370f7b8aeabbc317a4533 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sun, 20 Sep 2026 01:23:13 +0200 Subject: [PATCH 166/197] docs(openbao): document the kept recovery key Replaces the note that explained why the keys were discarded. Records where the key lives and why there rather than the inventory, what it does and does not protect, and the two costs: in swarm DIR_SECRETS is the NFS export, and the key is root-equivalent. Also corrects the MODE_RESET warning a second time. A reset is no longer a lockout, because the key survives it and rebuilds the AppRole. --- roles/web-app-openbao/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/web-app-openbao/README.md b/roles/web-app-openbao/README.md index a6be358039..f764f1d49c 100644 --- a/roles/web-app-openbao/README.md +++ b/roles/web-app-openbao/README.md @@ -108,7 +108,8 @@ docker run --rm -it \ - **The seal key is the whole ballgame.** The `static` seal encrypts the raft root key with a 32-byte key generated into the ansible-vault-encrypted inventory as `secrets.credentials.seal_key` and injected as `env://OPENBAO_SEAL_KEY`. A backup of the raft volume is **undecryptable without that key**, and the key is *not* covered by `svc-bkp-secrets-2-local` (whose sources are `DIR_SECRETS`, the root CA and the ACME material). The inventory is its authoritative home โ€” back the inventory up accordingly. Upstream considers a static seal appropriate only where an existing source of trust already holds the key; here that source is the encrypted inventory, at the same trust level as every other platform credential. - **The seal key is pinned, and that is deliberate.** `meta/secrets.yml` marks it `rotatable: false`, so the per-deploy rotation skips it. The static seal carries one generation of slack, and rotating on every deploy spends all of it continuously: a run that skips a generation, or dies between the two AppRole pins, leaves the raft store readable only from a volume backup plus the key that wrapped it โ€” and the recovery keys that would otherwise authorise a rekey were discarded at init. The key gains little from frequent rotation in exchange, because it lives in the vault-encrypted inventory either way, and an attacker holding that inventory holds the AppRole too and does not need the seal key to reach the data. Rotate it deliberately instead, by changing `secrets.credentials.seal_key` in the inventory: `templates/openbao.hcl.j2` then renders the `previous_key_id`/`previous_key` pair, the node unseals with the old key and rewraps to the new one, and the next deploy sees them equal again. The cost of pinning is that a leaked inventory backup stays valid against future volume snapshots instead of ageing out in two deploys. -- **Recovery keys are generated and then deliberately discarded.** Initialising with an auto-seal yields *recovery* keys rather than unseal shares. They cannot decrypt the root key โ€” only the seal key can โ€” so they are not disaster-recovery material for the data; they only authorise `operator generate-root` and `operator rekey`. Persisting them would mean writing OpenBao-generated secrets back into the inventory, which is exactly what the AppRole bootstrap exists to avoid, so the play does not keep them. **The consequence is explicit: the Ansible AppRole is the only administrative path into this instance.** If its RoleID/SecretID are changed out-of-band inside OpenBao, there is no way back in and the instance must be rebuilt from the volume backup plus the seal key. Re-running the role does not repair that, because reconciliation itself authenticates with the AppRole โ€” the rotation fallback below only covers the credentials the role itself applied. +- **The recovery key is kept, and it is the way back.** `bao operator init` issues it once, on the first deploy, and nothing ever reissues it โ€” which is exactly what makes it useful: it is the only authority here that does not rotate. `tasks/01_init.yml` persists it through `sys-token-store`, the platform role for "tokens that already exist (e.g. created by bootstrap processes)". That puts it in `DIR_SECRETS`, which outlives both `MODE_RESET` (it removes `DIR_COMPOSITIONS` only) and a replaced swarm manager (`/opt/compose` is manager-local), each of which takes `.applied.json` with it. `svc-bkp-secrets-2-local` already backs that directory up. The inventory would be the more natural home, but no role writes `host_vars` โ€” the inventory is written by the CLI, before the play. When neither the inventory AppRole nor the recorded one is accepted, `tasks/utils/recover.yml` mints a root token from the key via `bao operator generate-root` and hands over to `tasks/02_auth.yml`, which re-pins both halves to the inventory values. The deploy then continues instead of ending. Two costs are real: in swarm `DIR_SECRETS` is the NFS export, so the key is cluster-readable rather than manager-local, and a recovery key authorises `generate-root` and `rekey` โ€” whoever reads that file owns the instance. It sits beside `tokens.yml` and `databases.csv`, which are the same class of secret. +- **A recovery key cannot decrypt the store, so it is not a backup of the seal key.** Initialising with an auto-seal yields *recovery* keys rather than unseal shares. They cannot decrypt the root key โ€” only the seal key can โ€” so they are not disaster-recovery material for the data; they authorise `operator generate-root` and `operator rekey`, nothing more. Losing the seal key still means the raft volume is unreadable, whatever else survives. What the recovery key buys is administrative access, which is a different failure and the one the role could not previously survive: a RoleID/SecretID changed out-of-band inside OpenBao used to leave no way back in at all. - **Bootstrap leaves no stored root token.** The first deploy runs `bao operator init`, holds the root token in memory only, enables AppRole, pins both the RoleID and the SecretID to inventory-generated values via `auth/approle/role//role-id` and `.../custom-secret-id`, then revokes the root token. Every later deploy logs in with those inventory credentials, so nothing OpenBao generates is ever written back into the inventory. Tokens reach the CLI through stdin and the container token helper, never through process arguments, and the helper file is removed at the end of the run. - **Policies live outside the config directory.** OpenBao parses every file in `-config` as server configuration, so the RBAC policy `.hcl` files are mounted at `/openbao/policies/` instead of `/openbao/config/`. - **Storage path is `/openbao/file`, not `/openbao/data`.** The image runs as the non-root `openbao` user and pre-chowns only `/openbao/{config,logs,file}`. A fresh named volume inherits ownership from the image path, so raft can write there; a volume at `/openbao/data` would land root-owned and fail. @@ -125,7 +126,7 @@ docker run --rm -it \ - `tasks/01_init.yml` attempts the AppRole login with the inventory credentials, falls back to the recorded ones, and fails loudly when neither is accepted. Whichever one was accepted is the SecretID OpenBao actually holds, so that is what `tasks/06_rotate.yml` destroys after registering the new one โ€” rotation that only adds would revoke nothing. - `auth/approle/role//custom-secret-id` is **not** idempotent: writing a SecretID the role already holds answers `500 โ€ฆ SecretID is already registered`. The swap is therefore gated on the value having actually changed, which is also why the bootstrap in `tasks/02_auth.yml` pins directly instead of sharing the rotation path โ€” at that point nothing is registered yet. - `current_key_id` is derived as a short hash of the key rather than a counter. Upstream requires the identifier to match its key; a derived id cannot drift out of step with one, and no deploy has to know which generation it is. - The record is written last, after both pins, so a run that dies partway leaves the previous generation intact and the next deploy still authenticates. The one uncovered window is a failure *between* the two pins โ€” the RoleID re-pinned, the SecretID not, or the reverse โ€” which leaves a pairing neither the inventory nor the state file describes, and no recovery keys are kept to get back in. Two generations is the whole window for the AppRole. Discarding the state file while the volume survives puts the instance outside it: OpenBao still holds the previous SecretID, the inventory has already rotated past it, and the fallback has nothing left to read. `MODE_RESET` does exactly that โ€” `sys-svc-compose/tasks/01_reset.yml` removes `DIR_COMPOSITIONS` wholesale while the named volumes persist. The store itself stays readable, because the seal key is pinned and still comes from the inventory, but no administrative path into it remains. Purge the entity's volumes too when you reset โ€” the same asymmetry breaks every entity that keeps a credential inside its volume. + The record is written last, after both pins, so a run that dies partway leaves the previous generation intact and the next deploy still authenticates. The one uncovered window is a failure *between* the two pins โ€” the RoleID re-pinned, the SecretID not, or the reverse โ€” which leaves a pairing neither the inventory nor the state file describes. Two generations is the whole window for the AppRole, and discarding the state file while the volume survives puts the instance outside it: OpenBao still holds the previous SecretID, the inventory has already rotated past it, and the fallback has nothing left to read. `MODE_RESET` does exactly that โ€” `sys-svc-compose/tasks/01_reset.yml` removes `DIR_COMPOSITIONS` wholesale while the named volumes persist. Neither case is fatal any more: the store stays readable because the seal key is pinned, and the recovery key in `DIR_SECRETS` survives the reset and rebuilds the AppRole. Purge the entity's volumes too when you reset โ€” the same asymmetry breaks every entity that keeps a credential inside its volume, and those have no recovery key. - **Other entities still desynchronise the same way, and those are not defects.** A surviving `mariadb_data` fails its healthcheck with `Access denied for user 'root'@'127.0.0.1'` for exactly the reason above, without a rotation path of its own. CI never sees it because every run starts clean; locally, run `make compose-entity-purge apps=` before redeploying. - **A role-declared account cannot be logged in as unless it sets a password.** The tiers were first covered by two role-local accounts, `baooperator` and `baoreader`, which deployed and joined their RBAC groups correctly but returned 403 on every LDAP bind: a user declared without an explicit `password` inherits the `strong_password` fallback expression, which is re-evaluated on every render (`secrets.choice`, nothing persisted), so the value `svc-db-openldap` writes with `ldap_passwd` and the value `playwright.env.j2` renders can be different strings. Both accounts are gone now โ€” the tiers are covered by moving `biber` through the groups instead. [`f4398e1c5`](https://github.com/infinito-nexus/core/commit/f4398e1c58f3ce7d8d429472b2a023033242d747) closed the trap at the source: inventory creation now decides the value once, after the roles are resolved, so a declared user no longer carries a template that re-renders per use site. An explicit `password:`, as `user-administrator` and `web-app-semaphore` set, is therefore no longer required for a role-declared account to be authenticated as. - **Alert rules ship with the role.** `templates/prometheus/alert_rules.yml.j2` is discovered by the `alert_rule_roles` lookup and concatenated into the Prometheus rule file, so the seal alert lives next to the thing it watches. The fragment must start at the `groups:` item level (two-space-indented `- name:` entries). From 52c94416f07eabc4e7cff2d766a781bd257ac53c Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 05:51:15 +0000 Subject: [PATCH 167/197] update(galaxy): bump pinned Ansible collections --- requirements/requirements.galaxy.yml | 2 +- requirements/requirements.git.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/requirements.galaxy.yml b/requirements/requirements.galaxy.yml index 99944bdf14..30d8759605 100644 --- a/requirements/requirements.galaxy.yml +++ b/requirements/requirements.galaxy.yml @@ -13,4 +13,4 @@ collections: - name: community.library_inventory_filtering_v1 version: 1.1.5 - name: hetzner.hcloud - version: 7.0.1 + version: 7.1.0 diff --git a/requirements/requirements.git.yml b/requirements/requirements.git.yml index 5e6b9eaf0e..fbc7e57776 100644 --- a/requirements/requirements.git.yml +++ b/requirements/requirements.git.yml @@ -33,4 +33,4 @@ collections: - name: hetzner.hcloud source: https://github.com/ansible-collections/hetzner.hcloud.git type: git - version: 7.0.1 + version: 7.1.0 From 166e53066bec176da839bc088026f8c8a34026d4 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 06:03:28 +0000 Subject: [PATCH 168/197] update(images): update Docker image versions (svc-ai-ollama) --- roles/svc-ai-ollama/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/svc-ai-ollama/meta/services.yml b/roles/svc-ai-ollama/meta/services.yml index 49d885028a..751717156b 100644 --- a/roles/svc-ai-ollama/meta/services.yml +++ b/roles/svc-ai-ollama/meta/services.yml @@ -13,7 +13,7 @@ ollama: backup: no_stop_required: true image: ollama/ollama - version: 0.34.0 + version: 0.34.2 name: ollama min_storage: 3473MB cpus: "4.0" From bc9a9344e97692c0e6dcad32025ba870d157cbe1 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 06:03:40 +0000 Subject: [PATCH 169/197] update(images): update Docker image versions (svc-net-tor) --- roles/svc-net-tor/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/svc-net-tor/meta/services.yml b/roles/svc-net-tor/meta/services.yml index 0f872d03b9..8ea686d6c8 100644 --- a/roles/svc-net-tor/meta/services.yml +++ b/roles/svc-net-tor/meta/services.yml @@ -20,7 +20,7 @@ tor: # nocheck: category-modes exclusive: true primary: true image: debian - version: 13.6-slim + version: 13.7-slim custom: true name: tor node: "" From e82df4bc51199e59445f3057e8d55fe39ecdade3 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 06:03:51 +0000 Subject: [PATCH 170/197] update(images): update Docker image versions (web-app-erpnext) --- roles/web-app-erpnext/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-erpnext/meta/services.yml b/roles/web-app-erpnext/meta/services.yml index ae0a6e536b..fbddf587c1 100644 --- a/roles/web-app-erpnext/meta/services.yml +++ b/roles/web-app-erpnext/meta/services.yml @@ -65,7 +65,7 @@ erpnext: enabled: true shared: true image: frappe/erpnext - version: "v16.34.2" + version: "v16.35.0" name: erpnext min_storage: 1017MB ports: From 96d18cc6a970f4740f10590e62aa3879d1e6471a Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 06:04:05 +0000 Subject: [PATCH 171/197] update(images): update Docker image versions (web-app-jitsi) --- roles/web-app-jitsi/meta/services.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/web-app-jitsi/meta/services.yml b/roles/web-app-jitsi/meta/services.yml index b105470533..21051d1b6c 100644 --- a/roles/web-app-jitsi/meta/services.yml +++ b/roles/web-app-jitsi/meta/services.yml @@ -53,7 +53,7 @@ jitsi: swarm: enabled: true image: jitsi/web - version: "stable-9646" + version: "stable-11031" min_storage: 315MB # nocheck: default-placement-manager meta/volumes.yml uses type:bind config dirs staged on the stack host; a worker-scheduled task lacks the source paths (unpinning requires migrating them to nfs-backed named volumes) placement: manager @@ -87,7 +87,7 @@ prosody: port_key: http name: jitsi-prosody image: jitsi/prosody - version: "stable-9646" + version: "stable-11031" ports: internal: xmpp: 5222 @@ -102,7 +102,7 @@ prosody: jicofo: name: jitsi-jicofo image: jitsi/jicofo - version: "stable-9646" + version: "stable-11031" cpus: "1.0" mem_reservation: 512m mem_limit: 1g @@ -112,7 +112,7 @@ jicofo: jvb: name: jitsi-jvb image: jitsi/jvb - version: "stable-9646" + version: "stable-11031" cpus: "2.0" mem_reservation: 1g mem_limit: 2g From 8b2bc317839c3c5c2e15ff6cabcb07b3eea45a7d Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 06:04:09 +0000 Subject: [PATCH 172/197] update(images): update Docker image versions (web-app-keycloak) --- roles/web-app-keycloak/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-keycloak/meta/services.yml b/roles/web-app-keycloak/meta/services.yml index 7366782b3a..9625e85359 100644 --- a/roles/web-app-keycloak/meta/services.yml +++ b/roles/web-app-keycloak/meta/services.yml @@ -51,7 +51,7 @@ keycloak: shared: true provides: sso image: quay.io/keycloak/keycloak - version: "26.7.3" + version: "26.7.4" name: keycloak min_storage: 468MB cpus: "2.0" From 7ab0fec2a76835bf177cc1b63ec7d1eff457c281 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 06:04:21 +0000 Subject: [PATCH 173/197] update(images): update Docker image versions (web-app-opentalk) --- roles/web-app-opentalk/meta/services.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/web-app-opentalk/meta/services.yml b/roles/web-app-opentalk/meta/services.yml index a5e60c7b18..4e78c82a90 100644 --- a/roles/web-app-opentalk/meta/services.yml +++ b/roles/web-app-opentalk/meta/services.yml @@ -96,7 +96,7 @@ controller: livekit: name: opentalk-livekit image: livekit/livekit-server - version: v1.13.6 + version: v1.13.7 min_storage: 238MB cpus: "2.0" mem_reservation: 512m @@ -134,7 +134,7 @@ rabbit: retries: 5 name: opentalk-rabbit image: rabbitmq - version: 4.3.5 + version: 4.3.6 min_storage: 314MB cpus: "0.5" mem_reservation: 256m From 761c4a247dcaa71b8a07632c103aa73bc6545dd3 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 06:04:30 +0000 Subject: [PATCH 174/197] update(images): update Docker image versions (web-app-pgadmin) --- roles/web-app-pgadmin/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-pgadmin/meta/services.yml b/roles/web-app-pgadmin/meta/services.yml index 400e12388e..20b16398f4 100644 --- a/roles/web-app-pgadmin/meta/services.yml +++ b/roles/web-app-pgadmin/meta/services.yml @@ -11,7 +11,7 @@ pgadmin: # nocheck: default-placement-manager embedded sqlite config db; node-local in swarm placement: manager image: dpage/pgadmin4 - version: "9.17" + version: "9.18" min_storage: 378MB server_mode: false master_password_required: true From 6df80cac4a87f1516613026ec4cbfdbbf1b80a38 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 06:04:34 +0000 Subject: [PATCH 175/197] update(images): update Docker image versions (web-app-pihole) --- roles/web-app-pihole/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-pihole/meta/services.yml b/roles/web-app-pihole/meta/services.yml index 2abe5f7b63..e30e2bd67c 100644 --- a/roles/web-app-pihole/meta/services.yml +++ b/roles/web-app-pihole/meta/services.yml @@ -68,7 +68,7 @@ pihole: swarm: enabled: false image: pihole/pihole - version: "2026.07.2" + version: "2026.09.0" upstream_dns: "{{ NETWORK_PUBLIC_DNS_RESOLVERS }}" cpus: "0.5" mem_reservation: "128m" From 797bfacc2dfe129831ab1372f3016b3629408359 Mon Sep 17 00:00:00 2001 From: "infinito-nexus-updater[bot]" <286144613+infinito-nexus-updater[bot]@users.noreply.github.com> Date: Sun, 20 Sep 2026 06:04:37 +0000 Subject: [PATCH 176/197] update(images): update Docker image versions (web-app-prometheus) --- roles/web-app-prometheus/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-prometheus/meta/services.yml b/roles/web-app-prometheus/meta/services.yml index e2745043d5..54ace45103 100644 --- a/roles/web-app-prometheus/meta/services.yml +++ b/roles/web-app-prometheus/meta/services.yml @@ -79,7 +79,7 @@ alertmanager: flavor: wget path: -/healthy image: prom/alertmanager # nocheck: volumes-path-single-spot: the repository path of an image name, not a mount target - version: v0.34.0 + version: v0.34.1 name: prometheus-alertmanager cpus: "0.25" mem_reservation: 64m From 0071a6cde9d63bd7f055ddd04f85cf281e656ddb Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Sun, 20 Sep 2026 17:54:18 +0200 Subject: [PATCH 177/197] docs(mcp): regenerate the audit report after the baserow bump The merge from main moved web-app-baserow to 2.3.4 without regenerating the report the version is rendered into, so test_mcp_audit_report saw a document that no longer matched a fresh generation. Produced by make mcp-audit. --- docs/contributing/design/role/services/mcp/audit.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing/design/role/services/mcp/audit.md b/docs/contributing/design/role/services/mcp/audit.md index f9b15c0fa0..16d629ee0f 100644 --- a/docs/contributing/design/role/services/mcp/audit.md +++ b/docs/contributing/design/role/services/mcp/audit.md @@ -13,7 +13,7 @@ not the version the MCP surface was verified against. | role | version | source | transport | auth | subject | scope | tools | implementation | consumers | blocker | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | `svc-db-qdrant` | v1.19.1 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-svc-db-qdrant`) | read-only | 2 | adapter | `web-app-flowise`, `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | -| `web-app-baserow` | 2.3.3 | [link](https://baserow.io/user-docs/mcp-server) | streamable_http | bearer_token | service_account (`mcp-web-app-baserow`) | read-only | 4 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | +| `web-app-baserow` | 2.3.4 | [link](https://baserow.io/user-docs/mcp-server) | streamable_http | bearer_token | service_account (`mcp-web-app-baserow`) | read-only | 4 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-checkmk` | 2.4.0p32 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-checkmk`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-fider` | stable | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-fider`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-flowise` | 24-slim | [link](https://github.com/FlowiseAI/Flowise/tree/flowise%403.1.4/packages/server/src/routes/custom-mcp-servers) | โ€” | โ€” | โ€” | read-only | โ€” | native | โ€” | โ€” | From e06a62e19b1ed7140e6d714c46cf2a5fb5db4bca Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Mon, 21 Sep 2026 00:28:38 +0200 Subject: [PATCH 178/197] style(openbao): drop the narrative exception comments Review had these removed across the role once already, and the 11 September pass cleared every file that was flagged. The seal-key pin and the recovery-key work then reintroduced the same pattern in nine new places. Also removes two that predate the review and were never flagged: the healthcheck note in meta/services.yml and the entrypoint note in compose.yml.j2. Both are the same narrative-exception shape. Param docs, file headers and the plain contract notes in the Playwright specs stay: separate categories, none of them objected to. What the deleted comments recorded lives in the role README. --- roles/web-app-openbao/meta/services.yml | 1 - roles/web-app-openbao/tasks/00_core.yml | 3 --- roles/web-app-openbao/tasks/01_init.yml | 11 -------- .../web-app-openbao/tasks/04_auth_methods.yml | 1 - roles/web-app-openbao/tasks/06_rotate.yml | 11 +++----- roles/web-app-openbao/tasks/utils/recover.yml | 21 +++++----------- .../web-app-openbao/templates/compose.yml.j2 | 1 - roles/web-app-openbao/templates/proxy.conf.j2 | 1 - roles/web-app-openbao/vars/main.yml | 18 ++----------- .../test_credential_rotation.py | 25 ++++++++----------- 10 files changed, 22 insertions(+), 71 deletions(-) diff --git a/roles/web-app-openbao/meta/services.yml b/roles/web-app-openbao/meta/services.yml index bfe825ad8b..f020bbea99 100644 --- a/roles/web-app-openbao/meta/services.yml +++ b/roles/web-app-openbao/meta/services.yml @@ -67,7 +67,6 @@ openbao: http: 8093 healthcheck: flavor: wget - # Exception: an uninitialised container must report healthy or the init task can never reach it. path: "v1/sys/health?uninitcode=200&sealedcode=200&standbyok=true" run_after: - web-app-keycloak diff --git a/roles/web-app-openbao/tasks/00_core.yml b/roles/web-app-openbao/tasks/00_core.yml index fae44950be..1201b61e79 100644 --- a/roles/web-app-openbao/tasks/00_core.yml +++ b/roles/web-app-openbao/tasks/00_core.yml @@ -1,9 +1,6 @@ --- - include_tasks: utils/once/flag.yml -# Exception: must precede sys-stk-backend โ€” that role renders env.j2, which -# carries the previous seal key the node needs to unseal data the last deploy's -# key wrapped. Moving it after leaves the node sealed with no way back. - name: "๐Ÿ” Read the credentials the previous deploy applied" ansible.builtin.slurp: src: "{{ OPENBAO_APPLIED_STATE_HOST }}" diff --git a/roles/web-app-openbao/tasks/01_init.yml b/roles/web-app-openbao/tasks/01_init.yml index 93a405d416..2f268a1271 100644 --- a/roles/web-app-openbao/tasks/01_init.yml +++ b/roles/web-app-openbao/tasks/01_init.yml @@ -43,11 +43,6 @@ {{ (openbao_init_raw.stdout | from_json).root_token }} no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" - # Exception: the only authority here that never rotates. Without it the - # AppRole is the sole administrative path, so losing the recorded pair -- - # a reset, a replaced manager, a run that dies between the two pins in - # 06_rotate.yml -- locks the instance out for good while its data stays - # perfectly readable. - name: "๐Ÿ’พ Persist the recovery key through sys-token-store" ansible.builtin.include_role: name: sys-token-store @@ -96,9 +91,6 @@ delegate_to: "{{ OPENBAO_HOST_NODE }}" no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" -# Rationale: the credentials rotate every deploy, so the values this run was -# handed are not yet the ones pinned inside OpenBao. Fall back to what the -# previous deploy applied; 06_rotate.yml then re-pins to this run's values. - name: "๐Ÿ” Log in with the AppRole the previous deploy applied" ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node container exec -i {{ OPENBAO_LOCAL_CID | quote }} @@ -116,9 +108,6 @@ - openbao_approle_login.rc != 0 - OPENBAO_APPLIED.approle_role_id is defined -# Rationale: the AppRole is the only administrative path into the instance, so -# losing both halves of it used to end the deploy permanently. The recovery key -# does not rotate, which is what makes it a way back. - name: "๐Ÿš‘ Rebuild the AppRole from the recovery key when neither login is accepted" when: - openbao_approle_login.rc != 0 diff --git a/roles/web-app-openbao/tasks/04_auth_methods.yml b/roles/web-app-openbao/tasks/04_auth_methods.yml index 346a452543..ac49e7ab29 100644 --- a/roles/web-app-openbao/tasks/04_auth_methods.yml +++ b/roles/web-app-openbao/tasks/04_auth_methods.yml @@ -134,6 +134,5 @@ loop_var: openbao_group vars: alias_mount: "ldap/" - # Rationale: the RBAC LDAP layout contract encodes the application into the group CN. alias_name: "{{ application_id }}-{{ openbao_group.item }}" when: OPENBAO_LDAP_ENABLED | bool diff --git a/roles/web-app-openbao/tasks/06_rotate.yml b/roles/web-app-openbao/tasks/06_rotate.yml index f7ed0ba83a..481f61505f 100644 --- a/roles/web-app-openbao/tasks/06_rotate.yml +++ b/roles/web-app-openbao/tasks/06_rotate.yml @@ -1,10 +1,8 @@ --- # Re-pin the AppRole to the credentials this deploy was handed, then record what -# was applied. The platform rotates the AppRole per deploy, so the values the -# next deploy receives will differ from these; the state file is what lets that -# deploy still authenticate before it re-pins in turn. The seal key is pinned -# and normally repeats, and is recorded for the deploy that follows an operator -# changing it. +# was applied. The AppRole rotates per deploy, so the state file is what lets the +# next one authenticate before it re-pins in turn. The seal key is pinned and +# normally repeats; it is recorded for the deploy after an operator changes it. - name: "๐Ÿ”‘ Re-pin the RoleID to the value this deploy was handed" ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node container exec {{ OPENBAO_LOCAL_CID | quote }} @@ -13,9 +11,6 @@ changed_when: true delegate_to: "{{ OPENBAO_HOST_NODE }}" -# Exception: writing custom-secret-id with a SecretID the role already holds is a -# 500 "SecretID is already registered", not a no-op, so the swap has to be gated -# on the value actually having changed. - name: "๐Ÿ”‘ Register the SecretID this deploy was handed" ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node container exec -i {{ OPENBAO_LOCAL_CID | quote }} diff --git a/roles/web-app-openbao/tasks/utils/recover.yml b/roles/web-app-openbao/tasks/utils/recover.yml index 1b3e0ceab5..821ea6e762 100644 --- a/roles/web-app-openbao/tasks/utils/recover.yml +++ b/roles/web-app-openbao/tasks/utils/recover.yml @@ -1,21 +1,12 @@ --- -# Break-glass: mint a root token from the recovery key and rebuild the AppRole -# from the inventory. Reached only when neither the credentials this deploy was -# handed nor the ones the previous deploy recorded are accepted, which without -# this path ends the run with no administrative way into a node whose data is -# still perfectly readable. +# Break-glass: mint a root token from the recovery key, then hand to 02_auth.yml +# to re-pin the AppRole to the inventory values. Reached only when neither the +# credentials this deploy was handed nor the recorded ones are accepted. # -# The protocol is four steps and the token it yields is XOR-wrapped with a -# one-time password, so each step runs through the bao CLI and Ansible parses -# the result -- the image carries no python or jq to do it in-container. -# -# The last step hands over to 02_auth.yml, which re-pins both halves to the -# inventory values and revokes the root token it was given. The AppRole this -# deploy holds is therefore live on return, and 06_rotate.yml finds nothing -# left to swap. +# Each protocol step runs through the bao CLI and Ansible parses the result: the +# token comes back XOR-wrapped with a one-time password and the image carries +# neither python nor jq to unwrap it in-container. -# Exception: an attempt left behind by an interrupted run makes -init fail with -# "root generation already in progress". - name: "๐Ÿงน Cancel any root generation left in progress" ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node container exec {{ OPENBAO_LOCAL_CID | quote }} diff --git a/roles/web-app-openbao/templates/compose.yml.j2 b/roles/web-app-openbao/templates/compose.yml.j2 index 5f36f67fe0..637acc3d86 100644 --- a/roles/web-app-openbao/templates/compose.yml.j2 +++ b/roles/web-app-openbao/templates/compose.yml.j2 @@ -7,7 +7,6 @@ {% include 'roles/sys-svc-container/templates/base.yml.j2' %} {{ lookup('container_ports', ['openbao', 'http', DOCKER_BIND_HOST]) | indent(4) }} {{ lookup('container_healthcheck', service_name) | indent(4) }} - # Exception: no -config here; the image entrypoint appends -config=/openbao/config itself. command: - "server" {{ lookup('container_depends_on', application_id) | indent(4) }} diff --git a/roles/web-app-openbao/templates/proxy.conf.j2 b/roles/web-app-openbao/templates/proxy.conf.j2 index 75be4dbb20..97d5a2c23c 100644 --- a/roles/web-app-openbao/templates/proxy.conf.j2 +++ b/roles/web-app-openbao/templates/proxy.conf.j2 @@ -17,7 +17,6 @@ server {% include 'roles/sys-front-tls/templates/nginx_ssl_header.j2' %} - # Exception: the listener serves metrics unauthenticated for the internal scrape; this keeps them off the public surface. location = /v1/sys/metrics { deny all; } diff --git a/roles/web-app-openbao/vars/main.yml b/roles/web-app-openbao/vars/main.yml index 1564c6e4bc..8117f89516 100644 --- a/roles/web-app-openbao/vars/main.yml +++ b/roles/web-app-openbao/vars/main.yml @@ -20,23 +20,14 @@ OPENBAO_SEAL_KEY: "{{ lookup('config', application_id, 'secrets.credentials.seal OPENBAO_APPROLE_ROLE_ID: "{{ lookup('config', application_id, 'secrets.credentials.approle_role_id') }}" OPENBAO_APPROLE_SECRET_ID: "{{ lookup('config', application_id, 'secrets.credentials.approle_secret_id') }}" -# Rationale: the key id is derived from the key instead of counted โ€” upstream -# requires the identifier to match its key, and a derived id cannot drift out of -# step with one however rarely an operator rotates the key. OPENBAO_SEAL_KEY_ID: "{{ (OPENBAO_SEAL_KEY | hash('sha256'))[:16] }}" -# Rationale: what the previous deploy applied, read back before the config is -# rendered so the node can unseal with the old key and rewrap to the new one, and -# so the AppRole can still authenticate after its credentials rotated. OPENBAO_APPLIED_STATE_HOST: "{{ [lookup('container', application_id, 'directories.instance'), '.applied.json'] | path_join }}" OPENBAO_APPLIED: "{{ openbao_applied_state | default({}, true) }}" OPENBAO_SEAL_KEY_PREVIOUS: "{{ OPENBAO_APPLIED.seal_key | default('', true) }}" +OPENBAO_SEAL_KEY_PREVIOUS_ID: "{{ (OPENBAO_SEAL_KEY_PREVIOUS | hash('sha256'))[:16] if OPENBAO_SEAL_KEY_PREVIOUS else '' }}" +OPENBAO_SEAL_ROTATING: "{{ (OPENBAO_SEAL_KEY_PREVIOUS | length > 0) and (OPENBAO_SEAL_KEY_PREVIOUS != OPENBAO_SEAL_KEY) }}" -# Rationale: the recovery key comes from OpenBao at init rather than from the -# inventory generator, and sys-token-store is where the platform keeps exactly -# that class of secret. DIR_SECRETS outlives both MODE_RESET and a manager -# replacement, each of which takes .applied.json with it, and -# svc-bkp-secrets-2-local already covers it. OPENBAO_RECOVERY_USER_KEY: "administrator" OPENBAO_RECOVERY_TOKEN_APP: "{{ application_id }}-recovery" OPENBAO_RECOVERY_KEY: >- @@ -44,12 +35,7 @@ OPENBAO_RECOVERY_KEY: >- .get(OPENBAO_RECOVERY_USER_KEY, {}) .get('tokens', {}) .get(OPENBAO_RECOVERY_TOKEN_APP, '') }} -OPENBAO_SEAL_KEY_PREVIOUS_ID: "{{ (OPENBAO_SEAL_KEY_PREVIOUS | hash('sha256'))[:16] if OPENBAO_SEAL_KEY_PREVIOUS else '' }}" -OPENBAO_SEAL_ROTATING: "{{ (OPENBAO_SEAL_KEY_PREVIOUS | length > 0) and (OPENBAO_SEAL_KEY_PREVIOUS != OPENBAO_SEAL_KEY) }}" -# Rationale: resolved from the login rather than from the state file โ€” the -# SecretID OpenBao actually holds is whichever one was just accepted, and -# re-registering that same value answers 500, not a no-op. OPENBAO_SECRET_ID_REGISTERED: "{{ openbao_secret_id_registered | default('', true) }}" OPENBAO_SECRET_ID_ROTATING: "{{ (OPENBAO_SECRET_ID_REGISTERED | length > 0) and (OPENBAO_SECRET_ID_REGISTERED != OPENBAO_APPROLE_SECRET_ID) }}" OPENBAO_KV_MOUNT: "infinito" diff --git a/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py b/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py index 164b2a90cd..4beccfb7b3 100644 --- a/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py +++ b/tests/unit/python/roles/web-app-openbao/test_credential_rotation.py @@ -58,13 +58,11 @@ def tasks_of(name: str) -> list[dict]: class TestWhichCredentialsRotate(unittest.TestCase): """The seal key survives a deploy; the AppRole does not. - Rotating the seal key on every deploy spends the static seal's single - generation of slack for nothing. A run that skips a generation, or dies - between the two AppRole pins, leaves the raft store readable only from a - volume backup plus the key that wrapped it, and the role keeps no recovery - keys to fall back on. The AppRole is the opposite case: it is the live - authentication path, and absorbing its rotation is what the fallback login - in 01_init.yml exists for. + The static seal carries one generation of slack, and rotating every deploy + spends it continuously: a run that skips a generation leaves the raft store + readable only from a volume backup plus the key that wrapped it. The + AppRole is the opposite case -- the live authentication path, whose + rotation the fallback login in 01_init.yml exists to absorb. """ def setUp(self): @@ -254,13 +252,12 @@ class TestTheSecretIdSwapIsGuarded(unittest.TestCase): class TestTheRecoveryKeyIsKeptAndUsable(unittest.TestCase): """The one authority that does not rotate, and the way back it buys. - `bao operator init` issues a recovery key once and nothing else ever - reissues it. Dropping it leaves the AppRole as the only administrative - path, so a reset, a replaced manager, or a run that dies between the two - pins in 06_rotate.yml locks the instance out for good while its data stays - readable. It goes to sys-token-store rather than the inventory: no role - writes host_vars, and DIR_SECRETS outlives both DIR_COMPOSITIONS and the - node, and is covered by svc-bkp-secrets-2-local. + `bao operator init` issues a recovery key once and nothing reissues it. + Dropping it leaves the AppRole as the only administrative path, so a reset, + a replaced manager, or a run that dies between the two pins in + 06_rotate.yml locks the instance out for good while its data stays + readable. sys-token-store holds it rather than the inventory, which no role + writes. """ def setUp(self): From e300d3a0dc12b0e4b7e2470ac083785cd42971c5 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Mon, 21 Sep 2026 17:45:02 +0200 Subject: [PATCH 179/197] refactor(prometheus): filter alert rules by activation, not per fragment The lookup filtered on group_names, so each fragment had to re-check whether its app was actually scraped. Deployed and scraped are different conditions: metrics can be off on a deployed role, and a rule on a series nothing scrapes alerts on absent() forever. Intersecting with native_metrics_apps moves that check to one place, and every fragment from here on is plain rules with no guard. --- plugins/lookup/alert_rule_roles.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/plugins/lookup/alert_rule_roles.py b/plugins/lookup/alert_rule_roles.py index 05096c9ca5..dd4e139c8e 100644 --- a/plugins/lookup/alert_rule_roles.py +++ b/plugins/lookup/alert_rule_roles.py @@ -2,6 +2,7 @@ from __future__ import annotations from typing import Any +from ansible.plugins.loader import lookup_loader from ansible.plugins.lookup import LookupBase from utils.cache import ROLES_DIR @@ -11,14 +12,19 @@ ALERT_RULES_TEMPLATE = ("templates", "prometheus", "alert_rules.yml.j2") class LookupModule(LookupBase): """ - Return a sorted list of deployed role IDs that ship their own Prometheus - alert rules, i.e. every role in ``group_names`` that has a template at - ``roles//templates/prometheus/alert_rules.yml.j2``. + Return a sorted list of role IDs whose Prometheus alert rules belong in + this deploy: every role that ships a template at + ``roles//templates/prometheus/alert_rules.yml.j2`` and is actually + scraped, i.e. appears in ``native_metrics_apps``. Keeps alert rules at the role that owns the metric instead of collecting them in web-app-prometheus: a role declares what "unhealthy" means for itself, and the Prometheus role only concatenates what the play enabled. + Filtering on activation here rather than inside each fragment is what keeps + the fragments free of boilerplate. A rule that fires on a series nothing + scrapes is worse than no rule: it alerts on ``absent()`` forever. + Usage in a template: {% for role_id in lookup('alert_rule_roles') %} {% include 'roles/' + role_id + '/templates/prometheus/alert_rules.yml.j2' %} @@ -39,11 +45,16 @@ class LookupModule(LookupBase): raise ValueError("alert_rule_roles lookup takes no positional terms") vars_ = variables or getattr(self._templar, "available_variables", {}) or {} - group_names: list[str] = vars_.get("group_names", []) or [] + + scraped: list[str] = lookup_loader.get( + "native_metrics_apps", + loader=self._loader, + templar=getattr(self, "_templar", None), + ).run([], variables=vars_)[0] result = [ role_id - for role_id in sorted(set(group_names)) + for role_id in sorted(set(scraped)) if (ROLES_DIR / role_id).joinpath(*ALERT_RULES_TEMPLATE).exists() ] From 71e4437634471af6c92998b25f236ab77244f242 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Mon, 21 Sep 2026 17:45:13 +0200 Subject: [PATCH 180/197] refactor(openbao): drop the activation guard from the alert fragment The lookup now returns only scraped roles, so the fragment is rules and nothing else. --- roles/web-app-openbao/templates/prometheus/alert_rules.yml.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/web-app-openbao/templates/prometheus/alert_rules.yml.j2 b/roles/web-app-openbao/templates/prometheus/alert_rules.yml.j2 index b2ce3a9243..f944990828 100644 --- a/roles/web-app-openbao/templates/prometheus/alert_rules.yml.j2 +++ b/roles/web-app-openbao/templates/prometheus/alert_rules.yml.j2 @@ -1,4 +1,3 @@ -{% if 'web-app-openbao' in lookup('native_metrics_apps') %} - name: openbao_seal interval: 30s rules: @@ -13,4 +12,3 @@ OpenBao has not reported an active, unsealed core for more than 2 minutes. Secrets, PKI issuance and machine logins are unavailable while it is sealed. Check the container and confirm the static seal key reached its environment. -{% endif %} From 603233f89459f6ffbed235197a84ff2e78510c58 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Mon, 21 Sep 2026 17:45:26 +0200 Subject: [PATCH 181/197] test(prometheus): pin that an unscraped role contributes no alert rules Stubs native_metrics_apps and asserts the intersection, including the case the refactor exists for: a role that ships a fragment and is deployed but is not scraped must not reach the rule file. --- .../plugins/lookup/test_alert_rule_roles.py | 73 ++++++++++++------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/tests/unit/python/plugins/lookup/test_alert_rule_roles.py b/tests/unit/python/plugins/lookup/test_alert_rule_roles.py index 7de5d73d55..09953d117e 100644 --- a/tests/unit/python/plugins/lookup/test_alert_rule_roles.py +++ b/tests/unit/python/plugins/lookup/test_alert_rule_roles.py @@ -1,13 +1,20 @@ """Unit tests for plugins/lookup/alert_rule_roles.py. Mocks the Ansible-side machinery and asserts the plugin returns exactly the -deployed roles that ship a ``templates/prometheus/alert_rules.yml.j2``. +scraped roles that ship a ``templates/prometheus/alert_rules.yml.j2``. + +Activation is delegated to ``native_metrics_apps`` rather than re-derived here, +so these tests stub that lookup and assert the intersection. A role that is +deployed but not scraped must not contribute rules: its alert would fire on +``absent()`` forever. """ from __future__ import annotations +import tempfile import unittest import unittest.mock as mock +from pathlib import Path import plugins.lookup.alert_rule_roles as module from plugins.lookup.alert_rule_roles import LookupModule @@ -39,19 +46,26 @@ def _seed_roles(tmp_path, roles_with_template, roles_without_template=()): class TestAlertRuleRolesLookup(unittest.TestCase): def setUp(self): - self._tmp = mock.patch.object(module, "ROLES_DIR") - self.roles_dir = self._tmp.start() - self.addCleanup(self._tmp.stop) + self._roles_dir = mock.patch.object(module, "ROLES_DIR") + self.roles_dir = self._roles_dir.start() + self.addCleanup(self._roles_dir.stop) + + def _patch_scraped(self, scraped): + """Stand in for the native_metrics_apps lookup.""" + inner = mock.MagicMock() + inner.run.return_value = [list(scraped)] + loader = mock.patch.object(module, "lookup_loader") + started = loader.start() + started.get.return_value = inner + self.addCleanup(loader.stop) def test_raises_when_positional_terms_supplied(self): + self._patch_scraped([]) lm = _make_lookup({"group_names": []}) with self.assertRaises(ValueError): lm.run(["unexpected"], variables=lm._templar.available_variables) - def test_returns_only_deployed_roles_that_ship_a_template(self): - import tempfile - from pathlib import Path - + def test_returns_only_scraped_roles_that_ship_a_template(self): with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) _seed_roles( @@ -60,35 +74,40 @@ class TestAlertRuleRolesLookup(unittest.TestCase): roles_without_template=["web-app-matomo"], ) module.ROLES_DIR = tmp_path + self._patch_scraped(["web-app-matomo", "web-app-openbao"]) - vars_ = { - "group_names": [ - "web-app-matomo", - "web-app-openbao", - "web-app-never-deployed", - ] - } - lm = _make_lookup(vars_) - self.assertEqual(lm.run([], variables=vars_), [["web-app-openbao"]]) + lm = _make_lookup({}) + self.assertEqual(lm.run([], variables={}), [["web-app-openbao"]]) + + def test_a_deployed_role_that_is_not_scraped_contributes_no_rules(self): + """The whole point of moving the filter out of the fragments. + + svc-db-postgres ships a template here but native_metrics_apps does not + list it, so its rules must not reach the file. + """ + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + _seed_roles(tmp_path, roles_with_template=["svc-db-postgres"]) + module.ROLES_DIR = tmp_path + self._patch_scraped([]) + + lm = _make_lookup({}) + self.assertEqual(lm.run([], variables={}), [[]]) def test_result_is_sorted_and_deduplicated(self): - import tempfile - from pathlib import Path - with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) _seed_roles(tmp_path, roles_with_template=["b-role", "a-role"]) module.ROLES_DIR = tmp_path + self._patch_scraped(["b-role", "a-role", "b-role"]) - vars_ = {"group_names": ["b-role", "a-role", "b-role"]} - lm = _make_lookup(vars_) - self.assertEqual(lm.run([], variables=vars_), [["a-role", "b-role"]]) - - def test_returns_empty_list_without_group_names(self): - import tempfile - from pathlib import Path + lm = _make_lookup({}) + self.assertEqual(lm.run([], variables={}), [["a-role", "b-role"]]) + def test_returns_empty_list_when_nothing_is_scraped(self): with tempfile.TemporaryDirectory() as td: module.ROLES_DIR = Path(td) + self._patch_scraped([]) + lm = _make_lookup({}) self.assertEqual(lm.run([], variables={}), [[]]) From 6a01b190b026a7d79c283532ebad47f0545514a4 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Mon, 21 Sep 2026 17:45:34 +0200 Subject: [PATCH 182/197] chore(openbao): promote the role to beta Twelve of twelve deploy jobs green across compose and swarm, clearnet and tor, on every distro in the rotation. --- roles/web-app-openbao/meta/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/web-app-openbao/meta/services.yml b/roles/web-app-openbao/meta/services.yml index f020bbea99..1f6c913ce8 100644 --- a/roles/web-app-openbao/meta/services.yml +++ b/roles/web-app-openbao/meta/services.yml @@ -71,7 +71,7 @@ openbao: run_after: - web-app-keycloak - svc-db-openldap - lifecycle: alpha + lifecycle: beta # nocheck: playwright-service-flag # nocheck: variants-coverage โ€” sso-variant-tor requires the onion on in every variant, so the false polarity has no variant left to live in From 08855ff9f0400d013e52b759a64fdb0f5e2db659 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Mon, 21 Sep 2026 17:45:45 +0200 Subject: [PATCH 183/197] docs(openbao): cut the developer notes down to what gets read Eighteen bullets to ten. Merges the four seal-and-recovery notes into two, folds the three path traps into one, compresses the rotation block from six sub-bullets to a paragraph, and drops the note on role-declared account passwords entirely: the commit it was waiting on has landed, so the workaround it described is no longer needed. Nothing operational is lost. The seal key still says a raft backup is undecryptable without the inventory, and the recovery key still says where it lives and what it cannot do. --- roles/web-app-openbao/README.md | 34 ++++++++++----------------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/roles/web-app-openbao/README.md b/roles/web-app-openbao/README.md index f764f1d49c..d5226a022f 100644 --- a/roles/web-app-openbao/README.md +++ b/roles/web-app-openbao/README.md @@ -106,30 +106,16 @@ docker run --rm -it \ ## Developer Notes -- **The seal key is the whole ballgame.** The `static` seal encrypts the raft root key with a 32-byte key generated into the ansible-vault-encrypted inventory as `secrets.credentials.seal_key` and injected as `env://OPENBAO_SEAL_KEY`. A backup of the raft volume is **undecryptable without that key**, and the key is *not* covered by `svc-bkp-secrets-2-local` (whose sources are `DIR_SECRETS`, the root CA and the ACME material). The inventory is its authoritative home โ€” back the inventory up accordingly. Upstream considers a static seal appropriate only where an existing source of trust already holds the key; here that source is the encrypted inventory, at the same trust level as every other platform credential. -- **The seal key is pinned, and that is deliberate.** `meta/secrets.yml` marks it `rotatable: false`, so the per-deploy rotation skips it. The static seal carries one generation of slack, and rotating on every deploy spends all of it continuously: a run that skips a generation, or dies between the two AppRole pins, leaves the raft store readable only from a volume backup plus the key that wrapped it โ€” and the recovery keys that would otherwise authorise a rekey were discarded at init. The key gains little from frequent rotation in exchange, because it lives in the vault-encrypted inventory either way, and an attacker holding that inventory holds the AppRole too and does not need the seal key to reach the data. Rotate it deliberately instead, by changing `secrets.credentials.seal_key` in the inventory: `templates/openbao.hcl.j2` then renders the `previous_key_id`/`previous_key` pair, the node unseals with the old key and rewraps to the new one, and the next deploy sees them equal again. The cost of pinning is that a leaked inventory backup stays valid against future volume snapshots instead of ageing out in two deploys. -- **The recovery key is kept, and it is the way back.** `bao operator init` issues it once, on the first deploy, and nothing ever reissues it โ€” which is exactly what makes it useful: it is the only authority here that does not rotate. `tasks/01_init.yml` persists it through `sys-token-store`, the platform role for "tokens that already exist (e.g. created by bootstrap processes)". That puts it in `DIR_SECRETS`, which outlives both `MODE_RESET` (it removes `DIR_COMPOSITIONS` only) and a replaced swarm manager (`/opt/compose` is manager-local), each of which takes `.applied.json` with it. `svc-bkp-secrets-2-local` already backs that directory up. The inventory would be the more natural home, but no role writes `host_vars` โ€” the inventory is written by the CLI, before the play. When neither the inventory AppRole nor the recorded one is accepted, `tasks/utils/recover.yml` mints a root token from the key via `bao operator generate-root` and hands over to `tasks/02_auth.yml`, which re-pins both halves to the inventory values. The deploy then continues instead of ending. Two costs are real: in swarm `DIR_SECRETS` is the NFS export, so the key is cluster-readable rather than manager-local, and a recovery key authorises `generate-root` and `rekey` โ€” whoever reads that file owns the instance. It sits beside `tokens.yml` and `databases.csv`, which are the same class of secret. -- **A recovery key cannot decrypt the store, so it is not a backup of the seal key.** Initialising with an auto-seal yields *recovery* keys rather than unseal shares. They cannot decrypt the root key โ€” only the seal key can โ€” so they are not disaster-recovery material for the data; they authorise `operator generate-root` and `operator rekey`, nothing more. Losing the seal key still means the raft volume is unreadable, whatever else survives. What the recovery key buys is administrative access, which is a different failure and the one the role could not previously survive: a RoleID/SecretID changed out-of-band inside OpenBao used to leave no way back in at all. -- **Bootstrap leaves no stored root token.** The first deploy runs `bao operator init`, holds the root token in memory only, enables AppRole, pins both the RoleID and the SecretID to inventory-generated values via `auth/approle/role//role-id` and `.../custom-secret-id`, then revokes the root token. Every later deploy logs in with those inventory credentials, so nothing OpenBao generates is ever written back into the inventory. Tokens reach the CLI through stdin and the container token helper, never through process arguments, and the helper file is removed at the end of the run. -- **Policies live outside the config directory.** OpenBao parses every file in `-config` as server configuration, so the RBAC policy `.hcl` files are mounted at `/openbao/policies/` instead of `/openbao/config/`. -- **Storage path is `/openbao/file`, not `/openbao/data`.** The image runs as the non-root `openbao` user and pre-chowns only `/openbao/{config,logs,file}`. A fresh named volume inherits ownership from the image path, so raft can write there; a volume at `/openbao/data` would land root-owned and fail. -- **`command:` is just `server`.** The image entrypoint appends `-config=/openbao/config` itself; passing an explicit `-config` duplicates the flag. -- **CLI OIDC login is out of scope.** The `redirect_uris` lookup registers `https:///*` per SSO consumer, which covers the UI and API callbacks but not the CLI's `http://localhost:8250/oidc/callback`. Use AppRole for non-interactive access. -- **Visiting `/ui/vault/logout` does not end the session.** Measured against a live instance: after that navigation the UI returns to `/ui/vault/secrets` still authenticated, because the Ember app restores its token from browser storage. Use the UI's own logout control, or clear site data, when you actually need the session gone. -- **Do not alert on `vault_core_unsealed`.** OpenBao keeps Vault's `vault_*` metric namespace, and on a healthy unsealed node this gauge is emitted as `vault_core_unsealed{cluster=""} 0` โ€” an alert on `== 0` fires permanently. Use `/v1/sys/health` (503 while sealed) as the seal signal; `vault_core_active{cluster="โ€ฆ"} == 1` corroborates from the metrics side. -- **Swarm:** single replica, `placement: manager`, `nfs: false` on the raft volume and `service_update_order = stop-first` โ€” raft's BoltDB store is mmap + `fsync` based, so it must stay node-local and must never have two tasks writing the same directory. -- **The RBAC tiers are proven by moving `biber` through the groups, not by dedicated accounts.** `files/playwright/test-rbac-groups.js` signs the administrator into LAM, adds `biber` to `cn=web-app-openbao-,ou=roles` through LAM's Multi edit tool, logs `biber` into OpenBao over LDAP and asserts both the policy set and the capability that tier grants, then removes him again and asserts the revoke. One account covers all three tiers, and the grant and the revoke are observed rather than assumed โ€” a permanently-assigned member can only ever show a steady state. LAM's *group list* cannot be used for this: it renders `posixGroup` entries, while the RBAC role groups are `groupOfNames` under `ou=roles`. LAM applies the change asynchronously, so the spec polls the resulting policy set instead of LAM's own progress wording. -- **Every deploy rotates the AppRole, and the role is built around that.** The platform regenerates generated-algorithm credentials on each deploy โ€” that is how a changed Ansible password reaches the application instead of silently diverging from it. For OpenBao it means the `role_id`/`secret_id` a run is handed are *not* the ones the running instance was configured with, which would otherwise leave the AppRole answering `invalid role or secret ID`. The role therefore carries the transition itself: - - `tasks/06_rotate.yml` records what each deploy actually applied in `.applied.json` inside the compose project directory, `0600`, root-owned. The directory already holds the seal key in plaintext in `.env/env`, so this adds no new exposure class. - - `tasks/00_core.yml` reads that file back **before** `sys-stk-backend` renders `env.j2` โ€” the rendered env must already carry `OPENBAO_SEAL_KEY_PREVIOUS`, or the node comes up with no way to decrypt its own store. - - `templates/openbao.hcl.j2` emits the `previous_key_id`/`previous_key` pair only while the two keys differ. The static seal supports `n-1` โ†’ `n` rotation, so the node unseals with the old key and rewraps to the new one. - - `tasks/01_init.yml` attempts the AppRole login with the inventory credentials, falls back to the recorded ones, and fails loudly when neither is accepted. Whichever one was accepted is the SecretID OpenBao actually holds, so that is what `tasks/06_rotate.yml` destroys after registering the new one โ€” rotation that only adds would revoke nothing. - - `auth/approle/role//custom-secret-id` is **not** idempotent: writing a SecretID the role already holds answers `500 โ€ฆ SecretID is already registered`. The swap is therefore gated on the value having actually changed, which is also why the bootstrap in `tasks/02_auth.yml` pins directly instead of sharing the rotation path โ€” at that point nothing is registered yet. - - `current_key_id` is derived as a short hash of the key rather than a counter. Upstream requires the identifier to match its key; a derived id cannot drift out of step with one, and no deploy has to know which generation it is. - The record is written last, after both pins, so a run that dies partway leaves the previous generation intact and the next deploy still authenticates. The one uncovered window is a failure *between* the two pins โ€” the RoleID re-pinned, the SecretID not, or the reverse โ€” which leaves a pairing neither the inventory nor the state file describes. Two generations is the whole window for the AppRole, and discarding the state file while the volume survives puts the instance outside it: OpenBao still holds the previous SecretID, the inventory has already rotated past it, and the fallback has nothing left to read. `MODE_RESET` does exactly that โ€” `sys-svc-compose/tasks/01_reset.yml` removes `DIR_COMPOSITIONS` wholesale while the named volumes persist. Neither case is fatal any more: the store stays readable because the seal key is pinned, and the recovery key in `DIR_SECRETS` survives the reset and rebuilds the AppRole. Purge the entity's volumes too when you reset โ€” the same asymmetry breaks every entity that keeps a credential inside its volume, and those have no recovery key. -- **Other entities still desynchronise the same way, and those are not defects.** A surviving `mariadb_data` fails its healthcheck with `Access denied for user 'root'@'127.0.0.1'` for exactly the reason above, without a rotation path of its own. CI never sees it because every run starts clean; locally, run `make compose-entity-purge apps=` before redeploying. -- **A role-declared account cannot be logged in as unless it sets a password.** The tiers were first covered by two role-local accounts, `baooperator` and `baoreader`, which deployed and joined their RBAC groups correctly but returned 403 on every LDAP bind: a user declared without an explicit `password` inherits the `strong_password` fallback expression, which is re-evaluated on every render (`secrets.choice`, nothing persisted), so the value `svc-db-openldap` writes with `ldap_passwd` and the value `playwright.env.j2` renders can be different strings. Both accounts are gone now โ€” the tiers are covered by moving `biber` through the groups instead. [`f4398e1c5`](https://github.com/infinito-nexus/core/commit/f4398e1c58f3ce7d8d429472b2a023033242d747) closed the trap at the source: inventory creation now decides the value once, after the roles are resolved, so a declared user no longer carries a template that re-renders per use site. An explicit `password:`, as `user-administrator` and `web-app-semaphore` set, is therefore no longer required for a role-declared account to be authenticated as. -- **Alert rules ship with the role.** `templates/prometheus/alert_rules.yml.j2` is discovered by the `alert_rule_roles` lookup and concatenated into the Prometheus rule file, so the seal alert lives next to the thing it watches. The fragment must start at the `groups:` item level (two-space-indented `- name:` entries). +- **The seal key is the whole ballgame.** The `static` seal wraps the raft root key with `secrets.credentials.seal_key`, injected as `env://OPENBAO_SEAL_KEY`. A backup of the raft volume is **undecryptable without it**, and `svc-bkp-secrets-2-local` does not cover it โ€” the encrypted inventory is its only home, so back the inventory up accordingly. It is pinned `rotatable: false`: the static seal carries one generation of slack, and spending that every deploy makes any skipped generation unrecoverable. Rotate it deliberately by changing the inventory value; `openbao.hcl.j2` then emits the `previous_key`/`previous_key_id` pair, the node unseals with the old key and rewraps to the new one. +- **The recovery key is the way back in.** `bao operator init` issues it once and nothing reissues it, which is what makes it useful โ€” it is the only authority here that does not rotate. It is persisted through `sys-token-store` (`DIR_SECRETS`, backed up, and it survives both `MODE_RESET` and a replaced manager, each of which takes `.applied.json` with it). When neither the inventory AppRole nor the recorded one is accepted, `tasks/utils/recover.yml` mints a root token from it and re-pins the AppRole. It cannot decrypt the store โ€” only the seal key can โ€” so it is not a substitute for backing up the inventory. +- **Every deploy rotates the AppRole.** The `role_id`/`secret_id` a run is handed are not the ones the running instance holds. `tasks/01_init.yml` tries the inventory pair, falls back to the pair recorded in `.applied.json`, then `tasks/06_rotate.yml` re-pins to this run's values and destroys the old SecretID. The record is written last, so a run that dies partway leaves the previous generation intact. `custom-secret-id` is **not** idempotent โ€” re-writing a held SecretID answers `500` โ€” so the swap is gated on the value having changed. +- **Bootstrap leaves no stored root token.** The first deploy holds it in memory, pins both AppRole halves to inventory values, then revokes it. Tokens reach the CLI through stdin and the container token helper, never through process arguments, and the helper file is removed at the end of the run. +- **Paths that look wrong and are not.** Policies mount at `/openbao/policies/`, because OpenBao parses *every* file under `-config` as server configuration. Storage is `/openbao/file`, not `/openbao/data`: the image pre-chowns only `/openbao/{config,logs,file}` for the non-root `openbao` user, so a volume elsewhere lands root-owned and raft cannot write. `command:` is just `server` โ€” the entrypoint appends `-config` itself, and passing it duplicates the flag. +- **Do not alert on `vault_core_unsealed`.** OpenBao keeps Vault's metric namespace, and a healthy unsealed node emits `vault_core_unsealed{cluster=""} 0`, so an alert on `== 0` fires permanently. Use `/v1/sys/health` (503 while sealed); `vault_core_active{cluster="โ€ฆ"} == 1` corroborates. +- **Alert rules ship with the role.** `templates/prometheus/alert_rules.yml.j2` is discovered by the `alert_rule_roles` lookup, which returns only roles that are actually scraped, so the fragment carries no activation guard of its own. It must start at the `groups:` item level. +- **Swarm:** single replica, `placement: manager`, `nfs: false` on the raft volume, `service_update_order = stop-first`. Raft's BoltDB store is mmap + `fsync` based, so it must stay node-local and must never have two tasks writing one directory. +- **CLI OIDC login is out of scope.** `redirect_uris` registers `https:///*`, covering UI and API callbacks but not the CLI's `http://localhost:8250/oidc/callback`. Use AppRole for non-interactive access. +- **Test notes.** The RBAC tiers are proven by moving `biber` through the groups with LAM's Multi edit tool and asserting both the grant and the revoke, rather than by dedicated accounts. `/ui/vault/logout` does not end a session โ€” the Ember app restores its token from browser storage โ€” so use the UI's own logout control when the session must actually be gone. ### Backup and restore From f88aea547350a9ae6dbd2591b31062f6c8255bb0 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 21 Sep 2026 10:27:12 +0200 Subject: [PATCH 184/197] docs(mcp): regenerate the audit report for the baserow bump test_the_committed_report_is_freshly_generated compares the committed table against a fresh render, and baserow had moved to 2.3.4 in its services.yml while the table still read 2.3.3. The bump arrived from the image-version updater, which edits services.yml and nothing else. The report is derived from those files but is committed, so every version bump for an MCP-serving role leaves it stale until someone runs `make mcp-audit`. Co-Authored-By: Claude Opus 5 (1M context) --- docs/contributing/design/role/services/mcp/audit.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing/design/role/services/mcp/audit.md b/docs/contributing/design/role/services/mcp/audit.md index f9b15c0fa0..16d629ee0f 100644 --- a/docs/contributing/design/role/services/mcp/audit.md +++ b/docs/contributing/design/role/services/mcp/audit.md @@ -13,7 +13,7 @@ not the version the MCP surface was verified against. | role | version | source | transport | auth | subject | scope | tools | implementation | consumers | blocker | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | `svc-db-qdrant` | v1.19.1 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-svc-db-qdrant`) | read-only | 2 | adapter | `web-app-flowise`, `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | -| `web-app-baserow` | 2.3.3 | [link](https://baserow.io/user-docs/mcp-server) | streamable_http | bearer_token | service_account (`mcp-web-app-baserow`) | read-only | 4 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | +| `web-app-baserow` | 2.3.4 | [link](https://baserow.io/user-docs/mcp-server) | streamable_http | bearer_token | service_account (`mcp-web-app-baserow`) | read-only | 4 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-checkmk` | 2.4.0p32 | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-checkmk`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-fider` | stable | [link](https://s.infinito.nexus/code) | streamable_http | bearer_token | service_account (`mcp-web-app-fider`) | read-only | 3 | adapter | `web-app-hermes`, `web-app-openclaw`, `web-app-openwebui` | โ€” | | `web-app-flowise` | 24-slim | [link](https://github.com/FlowiseAI/Flowise/tree/flowise%403.1.4/packages/server/src/routes/custom-mcp-servers) | โ€” | โ€” | โ€” | read-only | โ€” | native | โ€” | โ€” | From 26bde378a16d2f13387b633bb5639d155bf2e6fa Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 21 Sep 2026 16:31:35 +0200 Subject: [PATCH 185/197] fix(lmstudio): bind the server beyond loopback so litellm can reach it Upstream re-pushed lmstudio/llmster-preview:cpu on 2026-09-18. The old build's daemon-run.sh wrote networkInterface 0.0.0.0 into .internal/http-server-config.json before starting the server. The new one drops that block, and its lms resolves the address as --bind, then LMS_SERVER_HOST, then 127.0.0.1, reading only the port from that file. The server therefore listened on loopback alone. The image healthcheck curls localhost and every role task drives lms through container exec, so svc-ai-lmstudio stayed green, while litellm's warm-up got Connect call failed on lmstudio:1234 and failed Mattermost#2, Matrix#2, XWiki#2, Moodle#2 and Zammad#2 in fork run 35545771696. LMS_SERVER_HOST is the documented knob and restores exactly the old bind. The port stays expose-only on the container network, so nothing new is published. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-lmstudio/templates/compose.yml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/svc-ai-lmstudio/templates/compose.yml.j2 b/roles/svc-ai-lmstudio/templates/compose.yml.j2 index 3e05c26133..ba14ab4e22 100644 --- a/roles/svc-ai-lmstudio/templates/compose.yml.j2 +++ b/roles/svc-ai-lmstudio/templates/compose.yml.j2 @@ -7,6 +7,8 @@ {{ lookup('compose_only', 'container_name', LMSTUDIO_CONTAINER) }} expose: - "{{ LMSTUDIO_PORT }}" + environment: + LMS_SERVER_HOST: "0.0.0.0" {{ lookup('container_volumes', service_name) | indent(4) }} {{ lookup('container_networks') | indent(4) }} From 5a651f05d534d83745ce3f50b1a4610839bfa56e Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 21 Sep 2026 16:32:35 +0200 Subject: [PATCH 186/197] test(lmstudio): probe the server on its network address, not loopback The role test asked for /v1/models on 127.0.0.1 from inside the container, the one address that still answered while the server was bound to loopback alone. It stayed green and the failure surfaced one role later as a 500 in litellm's warm-up. Step 3 now resolves the container's first IPv4 through hostname -I and queries that address, which a loopback-only bind refuses. It reads the interfaces directly, so it needs neither the service name nor swarm's route back through the service VIP, and compose and swarm take the same path. When the models never appear, one more request tells an unreachable server apart from an unindexed store. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-lmstudio/files/test/test.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/roles/svc-ai-lmstudio/files/test/test.sh b/roles/svc-ai-lmstudio/files/test/test.sh index c981fce35d..d30ca9b919 100644 --- a/roles/svc-ai-lmstudio/files/test/test.sh +++ b/roles/svc-ai-lmstudio/files/test/test.sh @@ -30,8 +30,10 @@ json_items() { python3 -c 'import json,sys; [print(x) for x in json.loads(sys.argv[1])]' "$1" } +server_url="" + serves_model() { - in_container curl -fsS --connect-timeout 5 --max-time 30 "http://127.0.0.1:${LMSTUDIO_PORT}/v1/models" 2>/dev/null | + in_container curl -fsS --connect-timeout 5 --max-time 30 "${server_url}/v1/models" 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print("\n".join(m.get("id","") for m in d.get("data",[])))' 2>/dev/null | grep -qxF "$1" } @@ -78,7 +80,13 @@ for i in "${!blobs[@]}"; do echo "[OK] ${blobs[$i]} present and matching" done -echo "=== 3. the daemon serves every declared model ===" +echo "=== 3. the daemon serves every declared model on its network address ===" +address="$(in_container hostname -I | tr ' ' '\n' | grep -m1 -E '^[0-9]+(\.[0-9]+){3}$')" +if [ -z "${address}" ]; then + echo "[FAIL] hostname -I reported no IPv4 address inside the container" >&2 + exit 1 +fi +server_url="http://${address}:${LMSTUDIO_PORT}" mapfile -t models < <(json_items "${LMSTUDIO_EXPECTED_MODELS}") for model in "${models[@]}"; do attempt=1 @@ -91,8 +99,13 @@ for model in "${models[@]}"; do [ "${attempt}" -le "${RETRIES}" ] && sleep "${SLEEP_SECONDS}" done if [ "${attempt}" -gt "${RETRIES}" ]; then - echo "[FAIL] ${model} is on disk but /v1/models never listed it" >&2 - echo " the daemon did not index the store it was given at startup" >&2 + if ! in_container curl -fsS --connect-timeout 5 --max-time 30 -o /dev/null "${server_url}/v1/models"; then + echo "[FAIL] ${server_url} does not answer" >&2 + echo " other containers cannot reach the server; check that LMS_SERVER_HOST binds it beyond loopback" >&2 + else + echo "[FAIL] ${model} is on disk but /v1/models never listed it" >&2 + echo " the daemon did not index the store it was given at startup" >&2 + fi failed=1 fi done From 600b4450f6bc7b46c84b498b86cc647da91ede7a Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 21 Sep 2026 18:20:10 +0200 Subject: [PATCH 187/197] fix(lmstudio): wait for the daemon to finish starting before the preload The preload ran lms get as soon as the container existed. On a slow runner that reaches the daemon while its delayed init is still running, and the daemon then comes up without .internal/temp. Every later model load fails, because the chat template is written with mkdtemp into that missing folder: Failed to load model "qwen2.5-0.5b-instruct". Error: ENOENT: no such file or directory, mkdtemp '/root/.lmstudio/.internal/temp/lmstudio-chat-template-...'. litellm's warm-up then returned HTTP 400 in Moodle#2 of run 35617847242, while a local deploy on faster hardware stayed green. Reproduced on fresh llmster containers capped at 0.3 CPU: an lms get issued during startup left .internal/temp absent; the same lms get issued after /lmstudio-greeting answered left it in place. The preload now waits for that endpoint, which only answers once lms server start ran, and daemon-run.sh starts the server only after the daemon's ready file exists. Co-Authored-By: Claude Opus 5 (1M context) --- roles/svc-ai-lmstudio/tasks/utils/preload/node.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml b/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml index b89e50e2e5..1fc150e6df 100644 --- a/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml +++ b/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml @@ -2,6 +2,16 @@ - name: "๐Ÿ” Resolve the container before preloading on '{{ lmstudio_node }}'" ansible.builtin.include_tasks: utils/cid.yml +- name: "โณ Wait until the LM Studio daemon has finished starting on '{{ lmstudio_node }}'" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec {{ LMSTUDIO_LOCAL_CID | quote }} curl -fsS -o /dev/null http://127.0.0.1:{{ LMSTUDIO_PORT }}/lmstudio-greeting + register: lmstudio_daemon_ready + changed_when: false + retries: 60 + delay: 5 + until: lmstudio_daemon_ready.rc == 0 + delegate_to: "{{ lmstudio_node }}" + - name: "๐Ÿ—„๏ธ Measure the space the model store offers on '{{ lmstudio_node }}'" ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node container exec {{ LMSTUDIO_LOCAL_CID | quote }} df -P -B1 {{ LMSTUDIO_MODELS_TARGET | quote }} From 9261ac9b0119e38019f6d6e2e6531988dc2ebeb2 Mon Sep 17 00:00:00 2001 From: evangelostsak Date: Mon, 21 Sep 2026 18:35:42 +0200 Subject: [PATCH 188/197] docs(web-app-lam): record why script-src-elem cannot be tightened The previous reason named only multi edit, which made the relaxation look specific to one tool and newly needed. Neither is true. LAM emits inline