diff --git a/cli/administration/deploy/dedicated/runner.py b/cli/administration/deploy/dedicated/runner.py index e388d7555f..d1816a0047 100644 --- a/cli/administration/deploy/dedicated/runner.py +++ b/cli/administration/deploy/dedicated/runner.py @@ -1,6 +1,6 @@ from __future__ import annotations -import datetime +import datetime as dt import json import os import subprocess @@ -33,7 +33,7 @@ def run_ansible_playbook( ansible_args: list[str] | None = None, ) -> None: """Run ansible-playbook with the given parameters and execution modes.""" - start_time = datetime.datetime.now(tz=datetime.UTC) + start_time = dt.datetime.now(tz=dt.UTC) print(f"\n▶️ Script started at: {start_time.isoformat()}\n", flush=True) if modes.get("MODE_CLEANUP", False): @@ -146,6 +146,6 @@ def run_ansible_playbook( ) sys.exit(2) - end_time = datetime.datetime.now(tz=datetime.UTC) + end_time = dt.datetime.now(tz=dt.UTC) print(f"\n✅ Script ended at: {end_time.isoformat()}\n") print(f"⏱️ Total execution time: {end_time - start_time}\n") diff --git a/cli/contributing/mirror/resolver/__main__.py b/cli/contributing/mirror/resolver/__main__.py index 01df2b1a08..4085479356 100644 --- a/cli/contributing/mirror/resolver/__main__.py +++ b/cli/contributing/mirror/resolver/__main__.py @@ -12,8 +12,9 @@ from utils.docker.image.discovery import iter_role_images def main() -> int: parser = argparse.ArgumentParser( description=( - "Emit the mirrors.yml mapping that redirects every role-declared " - "image (applications.*.services.*) to its GHCR mirror URI." + "Emit the mirrors.yml mapping that redirects every image a " + "non-eol role declares (applications.*.services.*) to its GHCR " + "mirror URI." ), ) parser.add_argument("--repo-root", default=".") diff --git a/cli/contributing/mirror/sync/__main__.py b/cli/contributing/mirror/sync/__main__.py index 2212f332db..d453732b46 100644 --- a/cli/contributing/mirror/sync/__main__.py +++ b/cli/contributing/mirror/sync/__main__.py @@ -53,8 +53,8 @@ def _throttle_before_next_copy( def main() -> int: parser = argparse.ArgumentParser( description=( - "Mirror every role-declared image and every image the CI runner " - "pulls itself to the configured GHCR registry, " + "Mirror every image a non-eol role declares and every image the " + "CI runner pulls itself to the configured GHCR registry, " "optionally throttled and optionally skipping already-mirrored " "tags." ), diff --git a/cli/contributing/mirror/wait/__main__.py b/cli/contributing/mirror/wait/__main__.py index 85769b2f7c..0820284b5c 100644 --- a/cli/contributing/mirror/wait/__main__.py +++ b/cli/contributing/mirror/wait/__main__.py @@ -23,8 +23,8 @@ def _validate_positive_int(value: str) -> int: def main() -> int: parser = argparse.ArgumentParser( description=( - "Wait until every role-declared image and every image the CI runner " - "pulls itself has been mirrored to the " + "Wait until every image a non-eol role declares and every image " + "the CI runner pulls itself has been mirrored to the " "configured GHCR registry. Used on fork pull_request runs that depend " "on a pull_request_target mirror job populating the destination first." ), diff --git a/cli/meta/ci/report_failures.py b/cli/meta/ci/report_failures.py index b7ada33826..9e7038667a 100644 --- a/cli/meta/ci/report_failures.py +++ b/cli/meta/ci/report_failures.py @@ -149,19 +149,21 @@ def _existing_issue(repo: str, role: str) -> int | None: return None -def report(run_id: str, repo: str) -> int: - jobs = json.loads( - _gh( - [ - "api", - "--paginate", - f"repos/{repo}/actions/runs/{run_id}/jobs", - "--jq", - "[.jobs[] | {name, conclusion}]", - ] - ) - or "[]" +def list_jobs(run_id: str, repo: str) -> list[dict]: + out = _gh( + [ + "api", + "--paginate", + f"repos/{repo}/actions/runs/{run_id}/jobs", + "--jq", + ".jobs[] | {name, conclusion}", + ] ) + return [json.loads(line) for line in out.splitlines() if line.strip()] + + +def report(run_id: str, repo: str) -> int: + jobs = list_jobs(run_id, repo) roles = failed_roles(jobs) if not roles: print("No failed deploy roles.") diff --git a/cli/meta/env/__main__.py b/cli/meta/env/__main__.py index d4f7e3764f..d1abfe9c6f 100644 --- a/cli/meta/env/__main__.py +++ b/cli/meta/env/__main__.py @@ -2,8 +2,10 @@ from __future__ import annotations +import argparse import os import sys +from pathlib import Path from utils.env.builder import build_env from utils.env.parser import parse_static_env_with_comments @@ -14,18 +16,34 @@ from . import PROJECT_ROOT as REPO_ROOT HOST_OWNED_KEY = "INFINITO_DOTENV_HOST_OWNED" # nocheck: dotenv-host-owned-marker Reason: set only by compose.yml inside the runner container; not a configuration key, a .env entry could freeze the host .env -def main() -> int: +def main(argv: list[str] | None = None) -> int: + """Generate the env set from default.env and the runtime context. + + Args: + argv: command-line arguments; ``--output PATH`` writes the set to PATH + instead of the repository's ``.env``, which then stays untouched. + + Returns: + 0 on success, 2 when default.env is missing. + """ + parser = argparse.ArgumentParser(prog="python -m cli.meta.env") + parser.add_argument("--output", type=Path) + args = parser.parse_args([] if argv is None else argv) static_path = REPO_ROOT / "default.env" - out_path = REPO_ROOT / ".env" + out_path = args.output or REPO_ROOT / ".env" + out_name = args.output or ".env" if not static_path.is_file(): print(f"ERROR: missing {static_path}", file=sys.stderr) return 2 - if out_path.is_file() and os.environ.get(HOST_OWNED_KEY) == "true": + if ( + args.output is None + and out_path.is_file() + and os.environ.get(HOST_OWNED_KEY) == "true" + ): print( - f"Kept {out_path.relative_to(REPO_ROOT)}: the host checkout owns it " - f"({HOST_OWNED_KEY})", + f"Kept {out_name}: the host checkout owns it ({HOST_OWNED_KEY})", file=sys.stderr, ) return 0 @@ -33,12 +51,9 @@ def main() -> int: static, static_comments = parse_static_env_with_comments(static_path) eb = build_env(static, repo_root=REPO_ROOT, comments=static_comments) write_dotenv(eb, out_path) - print( - f"Wrote {out_path.relative_to(REPO_ROOT)} ({len(eb.values)} variables)", - file=sys.stderr, - ) + print(f"Wrote {out_name} ({len(eb.values)} variables)", file=sys.stderr) return 0 if __name__ == "__main__": - sys.exit(main()) + sys.exit(main(sys.argv[1:])) diff --git a/docs/contributing/artefact/image/mirror.md b/docs/contributing/artefact/image/mirror.md index a0a8af6a47..e2f97cf848 100644 --- a/docs/contributing/artefact/image/mirror.md +++ b/docs/contributing/artefact/image/mirror.md @@ -32,7 +32,7 @@ Example: `ghcr.io/kevinveenbirkenbach/infinito-nexus-core/mirror/docker.io/nextc ### Image Discovery 🔍 -`utils/docker/image/discovery.py` discovers all role images via `iter_role_images()`. See [origin.md](origin.md) for the declaration format and supported registries. +`utils/docker/image/discovery.py` discovers the role images via `iter_role_images()`. It leaves out every role whose lifecycle is `eol` (`is_maintained()` in `utils/update/base.py`), so neither the mirror nor `mirrors.yml` covers those images. See [origin.md](origin.md) for the declaration format and supported registries. ### GHCRProvider 🐳 diff --git a/docs/contributing/environment/variables.md b/docs/contributing/environment/variables.md index a14e12cd88..6b47e34511 100644 --- a/docs/contributing/environment/variables.md +++ b/docs/contributing/environment/variables.md @@ -27,6 +27,7 @@ Already-set env vars take precedence over `default.env` defaults. - [`env/`](../../../utils/env/) (under `utils/`) holds the parser, runtime helpers, writer, and one handler module per dynamically-computed variable under [`handlers/`](../../../utils/env/handlers/). - [`env/`](../../../cli/meta/env/) (under `cli/meta/`) provides the CLI entry point invoked as `python -m cli.meta.env`. - [`load.sh`](../../../scripts/meta/env/load.sh) is the shell loader; it sources `.env` and auto-generates it on a cold clone. + When the exported `INFINITO_DISTRO` differs from the one `.env` was generated for, it loads a set generated for that distro through `python -m cli.meta.env --output ` and leaves `.env` as it is. ## Variables NOT in `default.env` diff --git a/plugins/lookup/current_play_redirect_domains.py b/plugins/lookup/current_play_redirect_domains.py index 2b2f8ff835..0c5e0abd6b 100644 --- a/plugins/lookup/current_play_redirect_domains.py +++ b/plugins/lookup/current_play_redirect_domains.py @@ -126,9 +126,10 @@ class LookupModule(LookupBase): DOMAIN_PRIMARY even when no app claims it. On onion nodes (``svc-net-tor`` serving the host with a provisioned ``services.tor.node``) the bare node-onion apex is scheduled as an - additional source targeting the same ``DOMAIN_HOMEPAGE``, which is - already family-correct there (onion-injected domain lookup); both - entries require ``web-opt-rdr-domains`` in + additional source targeting the same ``DOMAIN_HOMEPAGE`` when that + is an onion host; a clearnet ``DOMAIN_HOMEPAGE`` (no dashboard + deployed) gets no onion apex entry, so an onion vhost never + redirects to clearnet. Both entries require ``web-opt-rdr-domains`` in ``lookup('deployment').deployed`` so isolated CI variants do not schedule a redirect whose target is not served. 3. per-app alias-to-canonical mappings from @@ -210,7 +211,11 @@ class LookupModule(LookupBase): primary_redirects.append( {"source": domain_primary, "target": domain_homepage} ) - if node_onion and node_onion != domain_homepage: + if ( + node_onion + and node_onion != domain_homepage + and domain_homepage.endswith(".onion") + ): primary_redirects.append( {"source": node_onion, "target": domain_homepage} ) diff --git a/plugins/lookup/email.py b/plugins/lookup/email.py index 49e1dd2c75..eaf2af77b3 100644 --- a/plugins/lookup/email.py +++ b/plugins/lookup/email.py @@ -169,9 +169,6 @@ class LookupModule(LookupBase): return _as_bool(variables.get("TLS_ENABLED")) if short_key == "port": external = _as_bool(resolved.get("external")) - # SSO relay: port 25 is the only listener not requiring SMTP AUTH. - if self._provider_uses_sso_relay(variables): - return 25 if not external: return 25 ports = self._provider_ports(variables) @@ -182,8 +179,7 @@ class LookupModule(LookupBase): # binds the implicit-TLS client ports plus the MX, so it advertises # no `submission`; fall back to the MX port rather than a listener # nothing answers on (`auth` drops with it -- an MX takes local - # recipients without SMTP AUTH, the same property the SSO relay - # above relies on). + # recipients without SMTP AUTH). return ports.get("submission") or ports.get("smtp", 25) if short_key == "host": env = resolved.get("environment") @@ -203,8 +199,6 @@ class LookupModule(LookupBase): env = resolved.get("environment") if env in ("external_container", "localhost"): return False - if self._provider_uses_sso_relay(variables): - return False # The MX listener offers no SMTP AUTH: when `port` fell back to it # because the provider binds no plaintext submission, announcing # AUTH aborts msmtp with EX_UNAVAILABLE before it ever sends. @@ -219,11 +213,7 @@ class LookupModule(LookupBase): return "off" return "on" if _as_bool(resolved.get("tls")) else "plain" if short_key == "start_tls": - if not network( - network_of(resolved.get("host")) - ).tls or self._provider_onion_primary(variables): - return False - return self._provider_uses_sso_relay(variables) + return False if short_key == "smtp": return True if short_key == "from": @@ -258,8 +248,7 @@ class LookupModule(LookupBase): def _provider_services(self, variables: dict[str, Any]) -> dict[str, Any]: """The active provider's ``services`` block, or ``{}`` when it cannot be - resolved. Both the relay probe and the port probe read the provider's - own self-declaration, so the applications lookup is done once here.""" + resolved. The port probe reads the provider's own self-declaration.""" apps = ApplicationsLookup() apps._templar = getattr(self, "_templar", None) forwarded = { @@ -302,27 +291,6 @@ class LookupModule(LookupBase): continue return out - def _provider_uses_sso_relay(self, variables: dict[str, Any]) -> bool: - """True when the active provider self-declares SSO relay (via - ``services.sso.oidc.submission_via_relay``) and Keycloak is deployed.""" - group_names = variables.get("group_names") or [] - if "web-app-keycloak" not in group_names: - return False - services = self._provider_services(variables) - sso = services.get("sso") or {} if isinstance(services, dict) else {} - oidc = sso.get("oidc") or {} if isinstance(sso, dict) else {} - if not isinstance(oidc, dict): - return False - # A variant/inventory can pin the provider's sso.enabled to a literal - # false while submission_via_relay stays true in the role defaults — - # then the provider keeps password auth and never widens allowRelaying, - # so relay mode must be off. Untemplated Jinja (the role default gates - # on group_names) falls through to the Keycloak check above. - enabled = sso.get("enabled") if isinstance(sso, dict) else None - if enabled is not None and "{{" not in str(enabled) and not _as_bool(enabled): - return False - return _as_bool(oidc.get("submission_via_relay")) - def _provider_onion_primary(self, variables: dict[str, Any]) -> bool: """True when the provider's own host serves it onion-first. diff --git a/pyproject.toml b/pyproject.toml index bd4e1d5efc..77d63272ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ deploy = [ "requests", ] dev = [ - "ruff==0.16.10", + "ruff==0.17.0", "import-linter", "pre-commit", "git-maintainer-tools", diff --git a/roles/svc-ai-ollama/meta/services.yml b/roles/svc-ai-ollama/meta/services.yml index 77fc9801ed..84f50e6450 100644 --- a/roles/svc-ai-ollama/meta/services.yml +++ b/roles/svc-ai-ollama/meta/services.yml @@ -13,7 +13,7 @@ ollama: backup: no_stop_required: true image: ollama/ollama - version: 0.40.3 + version: 0.40.4 name: ollama min_storage: 3473MB cpus: "4.0" diff --git a/roles/svc-bkp-local-2-device/files/python/script.py b/roles/svc-bkp-local-2-device/files/python/script.py index bd8ed49054..222193d962 100755 --- a/roles/svc-bkp-local-2-device/files/python/script.py +++ b/roles/svc-bkp-local-2-device/files/python/script.py @@ -3,7 +3,7 @@ # subprocess calls against an attached backup device; every branch it has is a return # code from the host, so the assertions would only restate the mocks -import datetime +import datetime as dt import glob import os import subprocess @@ -42,8 +42,7 @@ def main(): print(f"previous versions path: {previous_version_path}") current_version_path = str( - Path(versions_path) - / datetime.datetime.now(tz=datetime.UTC).strftime("%Y%m%d%H%M%S") + Path(versions_path) / dt.datetime.now(tz=dt.UTC).strftime("%Y%m%d%H%M%S") ) print(f"current versions path: {current_version_path}") diff --git a/roles/svc-net-tor-smtp/meta/services.yml b/roles/svc-net-tor-smtp/meta/services.yml index c70fd26cd7..b47d780ddf 100644 --- a/roles/svc-net-tor-smtp/meta/services.yml +++ b/roles/svc-net-tor-smtp/meta/services.yml @@ -15,7 +15,7 @@ tor-smtp: # nocheck: category-modes # nocheck: default-placement-manager dials svc-net-tor's host-loopback SOCKS listener, which is manager-pinned; the gateway must share that node placement: manager image: python - version: 3.14-slim + version: 3.15-slim name: tor-smtp ports: internal: diff --git a/roles/svc-opt-keyboard-color/files/python/script.py b/roles/svc-opt-keyboard-color/files/python/script.py index a5e977eaec..b223dda10a 100644 --- a/roles/svc-opt-keyboard-color/files/python/script.py +++ b/roles/svc-opt-keyboard-color/files/python/script.py @@ -1,4 +1,4 @@ -import datetime +import datetime as dt import subprocess import sys @@ -51,7 +51,7 @@ def get_current_period(current_time, color_times): """ sorted_periods = sorted(color_times.items()) for i, (period_start_str, _colors) in enumerate(sorted_periods): - period_start_time = datetime.datetime.strptime( # noqa: DTZ007 HH:MM is local time-of-day; no timezone applies + period_start_time = dt.datetime.strptime( # noqa: DTZ007 HH:MM is local time-of-day; no timezone applies period_start_str, "%H:%M" ).time() if current_time <= period_start_time: @@ -71,15 +71,15 @@ def calculate_transition_ratio(current_time, start_time, end_time): Returns: float: The transition ratio between 0 and 1. """ - today = datetime.datetime.now().date() # noqa: DTZ005 user's local "today" for time-of-day color picking + today = dt.datetime.now().date() # noqa: DTZ005 user's local "today" for time-of-day color picking - start_timestamp = datetime.datetime.combine(today, start_time).timestamp() - end_timestamp = datetime.datetime.combine(today, end_time).timestamp() + start_timestamp = dt.datetime.combine(today, start_time).timestamp() + end_timestamp = dt.datetime.combine(today, end_time).timestamp() if start_timestamp == end_timestamp: return 0 - current_timestamp = datetime.datetime.combine(today, current_time).timestamp() + current_timestamp = dt.datetime.combine(today, current_time).timestamp() transition_duration = end_timestamp - start_timestamp time_since_start = current_timestamp - start_timestamp @@ -117,12 +117,12 @@ def main(vendor_and_product_id): "04:00": ("990000", "5bc0eb"), } - current_time = datetime.datetime.now().time() # noqa: DTZ005 user's local time-of-day for color picking + current_time = dt.datetime.now().time() # noqa: DTZ005 user's local time-of-day for color picking start_color, end_color = get_current_period(current_time, color_times) sorted_times = sorted(color_times.keys()) for time_str in [*sorted_times, sorted_times[0]]: - next_start_time_obj = datetime.datetime.strptime( # noqa: DTZ007 HH:MM is local time-of-day; no timezone applies + next_start_time_obj = dt.datetime.strptime( # noqa: DTZ007 HH:MM is local time-of-day; no timezone applies time_str, "%H:%M" ).time() if current_time < next_start_time_obj: @@ -133,7 +133,7 @@ def main(vendor_and_product_id): for time, colors in color_times.items() if colors == (start_color, end_color) ) - current_period_start_time_obj = datetime.datetime.strptime( # noqa: DTZ007 HH:MM is local time-of-day; no timezone applies + current_period_start_time_obj = dt.datetime.strptime( # noqa: DTZ007 HH:MM is local time-of-day; no timezone applies current_period_start_time_str, "%H:%M" ).time() diff --git a/roles/sys-service/handlers/refresh_systemctl_service.yml b/roles/sys-service/handlers/refresh_systemctl_service.yml index ac19505e55..6c8fb43bca 100644 --- a/roles/sys-service/handlers/refresh_systemctl_service.yml +++ b/roles/sys-service/handlers/refresh_systemctl_service.yml @@ -53,5 +53,6 @@ Unable to start service {{ system_service_name }}. See shared service diagnostics above. when: - not (system_service_suppress_flush | bool) - - (systemd_set_state.failed | default(false) | bool) or (system_service_result.stdout | default('') | trim) not in ['success', ''] or - (system_service_exec_status.stdout | default('') | trim) not in ['0', ''] + - (systemd_set_state.failed | default(false) | bool) or (system_service_state != omit and ( + (system_service_result.stdout | default('') | trim) not in ['success', ''] or + (system_service_exec_status.stdout | default('') | trim) not in ['0', ''])) diff --git a/roles/sys-svc-container/files/shell/apt_repo.sh b/roles/sys-svc-container/files/shell/apt_repo.sh index 2fbb3e696a..ce983387ee 100644 --- a/roles/sys-svc-container/files/shell/apt_repo.sh +++ b/roles/sys-svc-container/files/shell/apt_repo.sh @@ -1,14 +1,12 @@ #!/usr/bin/env bash set -euo pipefail -KEYRING_DIR="/etc/apt/keyrings" -LEGACY_KEY_PATH="${KEYRING_DIR}/docker.gpg" -KEY_PATH="${KEYRING_DIR}/docker.asc" +KEY_PATH="/etc/apt/keyrings/docker.asc" usage() { cat >&2 <<'EOF' Usage: - apt_repo.sh install-key + apt_repo.sh install-key [keyring-path] apt_repo.sh sanitize-sources [keep-canonical: 0|1] [keyring-path] apt_repo.sh ensure-key-and-sanitize [keep-canonical: 0|1] [keyring-path] @@ -62,33 +60,39 @@ PY install_key() { local distro_id="$1" + local key_path="$2" + local keyring_dir="" local changed=0 local tmp="" - if [[ ! -d "${KEYRING_DIR}" ]]; then + keyring_dir="$(dirname "${key_path}")" + if [[ ! -d "${keyring_dir}" ]]; then changed=1 - elif [[ "$(stat -c '%a' "${KEYRING_DIR}" 2>/dev/null || true)" != "755" ]]; then + elif [[ "$(stat -c '%a' "${keyring_dir}" 2>/dev/null || true)" != "755" ]]; then changed=1 fi - run_privileged install -m 0755 -d "${KEYRING_DIR}" + run_privileged install -m 0755 -d "${keyring_dir}" - if [[ -e "${LEGACY_KEY_PATH}" ]]; then - run_privileged rm -f "${LEGACY_KEY_PATH}" + if [[ -e "${keyring_dir}/docker.gpg" ]]; then + run_privileged rm -f "${keyring_dir}/docker.gpg" changed=1 fi tmp="$(mktemp)" trap 'rm -f "${tmp:-}"' EXIT if ! download_file "https://download.docker.com/linux/${distro_id}/gpg" "${tmp}" || [[ ! -s "${tmp}" ]]; then - echo "[ERROR] Failed to download the Docker GPG key from https://download.docker.com/linux/${distro_id}/gpg." >&2 - return 1 + if [[ ! -s "${key_path}" ]]; then + echo "[ERROR] Failed to download the Docker GPG key from https://download.docker.com/linux/${distro_id}/gpg." >&2 + return 1 + fi + echo "[WARN] Failed to refresh the Docker GPG key from https://download.docker.com/linux/${distro_id}/gpg; keeping ${key_path}." >&2 + elif [[ ! -f "${key_path}" ]] || ! cmp -s "${tmp}" "${key_path}"; then + run_privileged install -m 0644 "${tmp}" "${key_path}" + changed=1 fi - if [[ ! -f "${KEY_PATH}" ]] || ! cmp -s "${tmp}" "${KEY_PATH}"; then - run_privileged install -m 0644 "${tmp}" "${KEY_PATH}" - changed=1 - elif [[ "$(stat -c '%a' "${KEY_PATH}" 2>/dev/null || true)" != "644" ]]; then - run_privileged chmod 0644 "${KEY_PATH}" + if [[ "$(stat -c '%a' "${key_path}" 2>/dev/null || true)" != "644" ]]; then + run_privileged chmod 0644 "${key_path}" changed=1 fi @@ -175,7 +179,7 @@ ensure_key_and_sanitize() { local changed=0 local output="" - output="$(install_key "${distro_id}")" + output="$(install_key "${distro_id}" "${expected_keyring}")" if [[ "${output}" == *"changed=1"* ]]; then changed=1 fi @@ -197,7 +201,7 @@ case "${ACTION}" in usage exit 2 fi - install_key "${DISTRO_ID}" + install_key "${DISTRO_ID}" "${3:-${KEY_PATH}}" ;; sanitize-sources) DISTRO_ID="${2:-}" diff --git a/roles/test-e2e-playwright/meta/services.yml b/roles/test-e2e-playwright/meta/services.yml index 893702e0a6..3289d145ec 100644 --- a/roles/test-e2e-playwright/meta/services.yml +++ b/roles/test-e2e-playwright/meta/services.yml @@ -4,4 +4,4 @@ test-e2e-playwright: lifecycle: beta playwright: image: mcr.microsoft.com/playwright - version: "v1.64.0-noble" + version: "v1.64.1-noble" diff --git a/roles/web-app-baserow/meta/services.yml b/roles/web-app-baserow/meta/services.yml index 6fc42a226d..a149fb27b1 100644 --- a/roles/web-app-baserow/meta/services.yml +++ b/roles/web-app-baserow/meta/services.yml @@ -66,7 +66,7 @@ baserow: backup: no_stop_required: true image: baserow/baserow - version: 2.4.0 + version: 2.4.1 custom: true name: baserow min_storage: 785MB diff --git a/roles/web-app-bluesky/Administration.md b/roles/web-app-bluesky/Administration.md index 2f5829d46d..a610f909dd 100644 --- a/roles/web-app-bluesky/Administration.md +++ b/roles/web-app-bluesky/Administration.md @@ -1,10 +1,10 @@ # Administration -## create user via POST +## Create a user via POST ```bash curl -X POST https://your-pds-domain/xrpc/com.atproto.server.createAccount \ - --user "admin:$admin-password" + --user "admin:${PDS_ADMIN_PASSWORD}" \ -H "Content-Type: application/json" \ -d '{ "email": "user@example.com", @@ -16,13 +16,23 @@ curl -X POST https://your-pds-domain/xrpc/com.atproto.server.createAccount \ ## Use pdsadmin +```bash compose exec -it pds pdsadmin - compose exec -it pds pdsadmin account create-invite-code +``` + +## Roll back the PDS image + +The PDS migrates its account database forward when it starts. An older image refuses a database that a newer image has migrated and stops with `corrupted migrations: previously executed migration ... is missing`. + +- You MUST NOT lower `services.pds.version` on its own. +- To roll back, you MUST restore the `bluesky_pds_data` volume from the backup generation taken before the update and pin the old version in the same step. Follow [A single volume](../svc-bkp-volume-2-local/README.md#a-single-volume) for the restore. +- That backup generation exists only when `svc-bkp-volume-2-local` is deployed and `MODE_BACKUP` is enabled. +- The restore discards everything the PDS stored after the update. ## Debugging -- Websocket: [piehost.com/websocket-tester](https://piehost.com/websocket-tester) +- Websocket: [piehost.com](https://piehost.com/websocket-tester) - Instance: [bsky-debug.app](https://bsky-debug.app) -Initial setup keine top level domain +Initial setup: no top-level domain. diff --git a/roles/web-app-bluesky/files/playwright/_appview.js b/roles/web-app-bluesky/files/playwright/_appview.js new file mode 100644 index 0000000000..d5e89d07e9 --- /dev/null +++ b/roles/web-app-bluesky/files/playwright/_appview.js @@ -0,0 +1,133 @@ +const { expect } = require("@playwright/test"); +const { resolveTimeout } = require("./timeouts"); +const { apiFetchOnion, apiGetOnion } = require("./personas"); + +const LXM_PATTERN = /lxm|lexicon method/i; +const PROXIED_METHODS = ["app.bsky.feed.getTimeline", "app.bsky.notification.listNotifications"]; +const PERSONAL_DETAILS = "app.bsky.actor.defs#personalDetailsPref"; +const ADULT_BIRTHDATE = "1990-01-01"; + +async function expectPdsProxiesAppView(request, pdsBaseUrl, accessJwt) { + for (const nsid of PROXIED_METHODS) { + const response = await apiGetOnion(request, `${pdsBaseUrl}/xrpc/${nsid}`, { + headers: { authorization: `Bearer ${accessJwt}` }, + timeout: resolveTimeout(30_000), + }); + const body = await response.text(); + expect( + response.status(), + `PDS must proxy ${nsid} to PDS_BSKY_APP_VIEW_URL with a service JWT the AppView accepts (got body: ${body})`, + ).toBe(200); + } +} + +async function brokerAccessJwt(page) { + let accessJwt = ""; + await expect + .poll( + async () => { + accessJwt = await page + .evaluate(() => JSON.parse(localStorage.getItem("BSKY_STORAGE") || "{}").session?.currentAccount?.accessJwt || "") + .catch(() => ""); + return accessJwt; + }, + { timeout: resolveTimeout(60_000), message: "broker handoff left no accessJwt in BSKY_STORAGE" }, + ) + .not.toBe(""); + return accessJwt; +} + +async function readPreferences(request, pdsBaseUrl, accessJwt) { + const response = await apiGetOnion(request, `${pdsBaseUrl}/xrpc/app.bsky.actor.getPreferences`, { + headers: { authorization: `Bearer ${accessJwt}` }, + timeout: resolveTimeout(30_000), + }); + expect(response.status(), `PDS getPreferences must return 200 (got body: ${await response.text()})`).toBe(200); + return (await response.json()).preferences; +} + +async function restorePreferences(request, pdsBaseUrl, accessJwt, preferences) { + if (preferences.some((entry) => entry.$type === PERSONAL_DETAILS && entry.birthDate)) return; + const restore = apiFetchOnion(request, `${pdsBaseUrl}/xrpc/app.bsky.actor.putPreferences`, { + method: "POST", + headers: { authorization: `Bearer ${accessJwt}`, "content-type": "application/json" }, + data: { preferences }, + timeout: resolveTimeout(30_000), + }).then((response) => { + if (response.status() !== 200) throw new Error(`putPreferences answered ${response.status()}`); + }); + const deadline = new Promise((_, reject) => { + setTimeout(() => reject(new Error("putPreferences timed out")), resolveTimeout(45_000)); + }); + await Promise.race([restore, deadline]).catch((err) => console.warn(`bluesky: preferences not restored: ${err.message}`)); +} + +function collectAppViewResponses(page) { + const seen = []; + page.on("response", (response) => { + const { pathname } = new URL(response.url()); + if (!pathname.startsWith("/xrpc/app.bsky.") || response.request().method() !== "GET") return; + const status = response.status(); + seen.push({ + nsid: pathname.slice("/xrpc/".length), + status, + body: status === 200 ? Promise.resolve("") : response.text().catch((err) => ``), + }); + }); + return seen; +} + +async function waitForResponseTo(seen, nsid, from = 0) { + await expect + .poll(() => seen.slice(from).some((entry) => entry.nsid === nsid), { + timeout: resolveTimeout(60_000), + message: `social-app never requested ${nsid}`, + }) + .toBe(true); +} + +async function expectNoFailures(seen, nsid) { + const failed = seen.filter((entry) => entry.nsid === nsid && entry.status !== 200); + expect( + await Promise.all(failed.map(async (entry) => `${entry.status} ${await entry.body}`)), + `PDS must proxy ${nsid} to the AppView with a service JWT the AppView accepts`, + ).toEqual([]); +} + +async function expectNoLxmRejections(seen, pageErrors) { + const lines = await Promise.all(seen.map(async (entry) => `${entry.nsid} ${entry.status} ${await entry.body}`)); + expect( + lines.filter((line) => LXM_PATTERN.test(line)), + "no AppView call may be rejected for a missing lxm claim", + ).toEqual([]); + expect( + pageErrors.filter((message) => LXM_PATTERN.test(message)), + "social-app must not raise lxm page errors after login", + ).toEqual([]); +} + +async function reachFollowingFeed(page) { + const home = page.getByTestId("followingFeedPage"); + const birthdateGate = page.getByRole("button", { name: "Click here to update your birthdate" }).first(); + await expect(async () => { + if (await birthdateGate.isVisible()) { + await birthdateGate.click(); + const dialog = page.getByTestId("birthDateSettingsDialog"); + await dialog.getByTestId("birthdayInput").fill(ADULT_BIRTHDATE); + await dialog.getByRole("button", { name: "Save birthdate" }).click(); + } + await expect(home).toBeVisible({ timeout: resolveTimeout(5_000) }); + }).toPass({ timeout: resolveTimeout(120_000) }); +} + +module.exports = { + brokerAccessJwt, + collectAppViewResponses, + expectNoFailures, + expectNoLxmRejections, + expectPdsProxiesAppView, + reachFollowingFeed, + readPreferences, + restorePreferences, + waitForResponseTo, +}; diff --git a/roles/web-app-bluesky/files/playwright/_shared.js b/roles/web-app-bluesky/files/playwright/_shared.js index c8e0b6698f..a771f91692 100644 --- a/roles/web-app-bluesky/files/playwright/_shared.js +++ b/roles/web-app-bluesky/files/playwright/_shared.js @@ -1,7 +1,9 @@ const { expect } = require("@playwright/test"); const { skipUnlessServiceEnabled } = require("./service-gating"); +const { resolveTimeout } = require("./timeouts"); const { decodeDotenvQuotedValue, + gotoOnion, normalizeBaseUrl, performKeycloakLoginForm, runAdminFlow, @@ -23,6 +25,36 @@ async function beforeEach({ page }) { await page.context().clearCookies(); } +async function loginThroughBroker(page) { + expect(adminUsername, "ADMIN_USERNAME must be set when OIDC is enabled").toBeTruthy(); + expect(adminPassword, "ADMIN_PASSWORD must be set when OIDC is enabled").toBeTruthy(); + expect(oidcIssuerUrl, "OIDC_ISSUER_URL must be set when OIDC is enabled").toBeTruthy(); + + const expectedOidcAuthUrl = `${oidcIssuerUrl}/protocol/openid-connect/auth`; + const expectedBaseUrl = baseUrl.replace(/\/$/, ""); + + await gotoOnion(page, `${expectedBaseUrl}/`); + + await expect + .poll(() => page.url(), { + timeout: resolveTimeout(60_000), + message: `expected redirect to Keycloak OIDC auth (${expectedOidcAuthUrl})`, + }) + .toContain(expectedOidcAuthUrl); + + await performKeycloakLoginForm(page, adminUsername, adminPassword); + + await expect + .poll(() => page.url(), { + timeout: resolveTimeout(90_000), + message: `expected post-OIDC URL to land on Bluesky outside /oauth2/* (got: ${page.url()})`, + }) + .toMatch(new RegExp(`^${expectedBaseUrl.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/(?!oauth2/)`)); + + await page.waitForLoadState("networkidle", { timeout: resolveTimeout(30_000) }).catch(() => {}); + await expect(page.locator("body")).toBeVisible({ timeout: resolveTimeout(60_000) }); +} + module.exports = { env: { baseUrl, @@ -34,6 +66,7 @@ module.exports = { pdsBaseUrl, }, skipUnlessServiceEnabled, + loginThroughBroker, performKeycloakLoginForm, runAdminFlow, runBiberFlow, diff --git a/roles/web-app-bluesky/files/playwright/playwright.spec.js b/roles/web-app-bluesky/files/playwright/playwright.spec.js index 3aef5c3604..419d563f3e 100644 --- a/roles/web-app-bluesky/files/playwright/playwright.spec.js +++ b/roles/web-app-bluesky/files/playwright/playwright.spec.js @@ -8,6 +8,9 @@ test.beforeEach(shared.beforeEach); require("./test-baseline").register(shared); require("./test-oidc-broker-handoff").register(shared); +require("./test-appview-oidc-api").register(shared); +require("./test-appview-oidc-render").register(shared); +require("./test-appview-native").register(shared); require("./test-native-admin").register(shared); require("./test-guest-persona").register(shared); require("./test-biber-persona").register(shared); diff --git a/roles/web-app-bluesky/files/playwright/test-appview-native.js b/roles/web-app-bluesky/files/playwright/test-appview-native.js new file mode 100644 index 0000000000..17e944015a --- /dev/null +++ b/roles/web-app-bluesky/files/playwright/test-appview-native.js @@ -0,0 +1,25 @@ +const { test, expect } = require("@playwright/test"); +const { resolveTimeout } = require("./timeouts"); +const { apiFetchOnion } = require("./personas"); +const { skipUnlessServiceDisabled } = require("./service-gating"); +const { expectPdsProxiesAppView } = require("./_appview"); + +exports.register = function (shared) { + test("native: PDS proxies Following and Notifications to its configured AppView with an lxm-bound service JWT (no oauth2)", async ({ page }) => { + skipUnlessServiceDisabled("sso"); + const { adminHandle, adminPassword, pdsBaseUrl } = shared.env; + expect(adminHandle, "ADMIN_HANDLE must be set when oauth2 is disabled").toBeTruthy(); + expect(adminPassword, "ADMIN_PASSWORD must be set when oauth2 is disabled").toBeTruthy(); + expect(pdsBaseUrl, "PDS_BASE_URL must be set when oauth2 is disabled").toBeTruthy(); + + const session = await apiFetchOnion(page.request, `${pdsBaseUrl}/xrpc/com.atproto.server.createSession`, { + method: "POST", + data: { identifier: adminHandle, password: adminPassword }, + timeout: resolveTimeout(30_000), + }); + expect(session.status(), `PDS createSession must return 200 (got body: ${await session.text()})`).toBe(200); + const { accessJwt } = await session.json(); + + await expectPdsProxiesAppView(page.request, pdsBaseUrl, accessJwt); + }); +}; diff --git a/roles/web-app-bluesky/files/playwright/test-appview-oidc-api.js b/roles/web-app-bluesky/files/playwright/test-appview-oidc-api.js new file mode 100644 index 0000000000..91ab38ec50 --- /dev/null +++ b/roles/web-app-bluesky/files/playwright/test-appview-oidc-api.js @@ -0,0 +1,14 @@ +const { test } = require("@playwright/test"); +const { resolveTimeout } = require("./timeouts"); +const { brokerAccessJwt, expectPdsProxiesAppView } = require("./_appview"); + +exports.register = function (shared) { + test("OIDC: PDS proxies Following and Notifications to the AppView for the broker session", async ({ page }) => { + shared.skipUnlessServiceEnabled("sso"); + test.setTimeout(resolveTimeout(480_000)); + + await shared.loginThroughBroker(page); + const accessJwt = await brokerAccessJwt(page); + await expectPdsProxiesAppView(page.request, shared.env.pdsBaseUrl, accessJwt); + }); +}; diff --git a/roles/web-app-bluesky/files/playwright/test-appview-oidc-render.js b/roles/web-app-bluesky/files/playwright/test-appview-oidc-render.js new file mode 100644 index 0000000000..c87c96954e --- /dev/null +++ b/roles/web-app-bluesky/files/playwright/test-appview-oidc-render.js @@ -0,0 +1,58 @@ +const { test, expect } = require("@playwright/test"); +const { isOnionTarget, resolveTimeout } = require("./timeouts"); +const { gotoOnion } = require("./personas"); +const { + brokerAccessJwt, + collectAppViewResponses, + expectNoFailures, + expectNoLxmRejections, + reachFollowingFeed, + readPreferences, + restorePreferences, + waitForResponseTo, +} = require("./_appview"); + +exports.register = function (shared) { + test("OIDC: social-app renders Following and Notifications without errorMessageView or lxm page errors", async ({ page }) => { + shared.skipUnlessServiceEnabled("sso"); + test.skip( + isOnionTarget(), + "social-app only boots after loading its age-assurance config from public.api.bsky.app, which a Tor-routed test browser cannot reliably reach", + ); + test.setTimeout(resolveTimeout(900_000)); + + const pageErrors = []; + page.on("pageerror", (err) => pageErrors.push(err.message)); + const seen = collectAppViewResponses(page); + const { pdsBaseUrl } = shared.env; + + await shared.loginThroughBroker(page); + const accessJwt = await brokerAccessJwt(page); + const preferences = await readPreferences(page.request, pdsBaseUrl, accessJwt); + try { + await waitForResponseTo(seen, "app.bsky.notification.listNotifications"); + await expectNoLxmRejections(seen, pageErrors); + + await reachFollowingFeed(page); + await waitForResponseTo(seen, "app.bsky.feed.getTimeline"); + await expectNoFailures(seen, "app.bsky.feed.getTimeline"); + const following = page.getByTestId("followingFeedPage"); + await expect(following.locator('[data-testid^="feedItem-by-"]').or(following.getByTestId("errorMessageView")).first()).toBeVisible({ + timeout: resolveTimeout(60_000), + }); + await expect(following.getByTestId("errorMessageView"), "Following must render without an errorMessageView").toHaveCount(0); + + const notificationsFrom = seen.length; + await gotoOnion(page, `${shared.env.baseUrl}/notifications`); + const notifications = page.getByTestId("notificationsScreen"); + await expect(notifications).toBeVisible({ timeout: resolveTimeout(60_000) }); + await waitForResponseTo(seen, "app.bsky.notification.listNotifications", notificationsFrom); + await expectNoFailures(seen, "app.bsky.notification.listNotifications"); + await expect(notifications.getByTestId("errorMessageView"), "Notifications must render without an errorMessageView").toHaveCount(0); + + await expectNoLxmRejections(seen, pageErrors); + } finally { + await restorePreferences(page.request, pdsBaseUrl, accessJwt, preferences); + } + }); +}; diff --git a/roles/web-app-bluesky/files/playwright/test-oidc-broker-handoff.js b/roles/web-app-bluesky/files/playwright/test-oidc-broker-handoff.js index eb819c61eb..e7af8f9962 100644 --- a/roles/web-app-bluesky/files/playwright/test-oidc-broker-handoff.js +++ b/roles/web-app-bluesky/files/playwright/test-oidc-broker-handoff.js @@ -1,6 +1,4 @@ const { test, expect } = require("@playwright/test"); -const { resolveTimeout } = require("./timeouts"); -const { gotoOnion } = require("./personas"); exports.register = function (shared) { test("OIDC: oauth2-proxy + login-broker drop a Bluesky session into social-app via Keycloak (variant A+)", async ({ page }) => { @@ -18,44 +16,7 @@ exports.register = function (shared) { // without ever showing the synthesised app-password to the // browser. shared.skipUnlessServiceEnabled("sso"); - const { baseUrl, oidcIssuerUrl, adminUsername, adminPassword } = shared.env; - expect(adminUsername, "ADMIN_USERNAME must be set when OIDC is enabled").toBeTruthy(); - expect(adminPassword, "ADMIN_PASSWORD must be set when OIDC is enabled").toBeTruthy(); - expect(oidcIssuerUrl, "OIDC_ISSUER_URL must be set when OIDC is enabled").toBeTruthy(); - - const expectedOidcAuthUrl = `${oidcIssuerUrl}/protocol/openid-connect/auth`; - const expectedBaseUrl = baseUrl.replace(/\/$/, ""); - - await gotoOnion(page, `${expectedBaseUrl}/`); - - await expect - .poll(() => page.url(), { - timeout: resolveTimeout(60_000), - message: `expected redirect to Keycloak OIDC auth (${expectedOidcAuthUrl})`, - }) - .toContain(expectedOidcAuthUrl); - - await shared.performKeycloakLoginForm(page, adminUsername, adminPassword); - - // Wait for the post-OIDC redirect chain to settle. After Keycloak - // accepts the credentials, the browser bounces through - // /oauth2/callback → / → broker handoff → social-app - // and we want the URL to come to rest on the Bluesky base URL — - // NOT on an `/oauth2/*` path. The `expect.poll` form alone matches - // any URL containing the base, including the intermediate - // `/oauth2/start` redirect, so we additionally pin the path to - // not be under `/oauth2/`. - await expect - .poll(() => page.url(), { - timeout: resolveTimeout(90_000), - message: `expected post-OIDC URL to land on Bluesky outside /oauth2/* (got: ${page.url()})`, - }) - .toMatch(new RegExp(`^${expectedBaseUrl.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/(?!oauth2/)`)); - - // Wait for the network to settle so the broker's handoff JS can - // commit localStorage + cookie before we inspect them. - await page.waitForLoadState("networkidle", { timeout: resolveTimeout(30_000) }).catch(() => {}); - await expect(page.locator("body")).toBeVisible({ timeout: resolveTimeout(60_000) }); + await shared.loginThroughBroker(page); // The broker writes a schema-compliant `BSKY_STORAGE` payload // built from upstream `social-app@1.121.0`'s persisted defaults diff --git a/roles/web-app-bluesky/meta/services.yml b/roles/web-app-bluesky/meta/services.yml index 6c4af68bfa..13545fe059 100644 --- a/roles/web-app-bluesky/meta/services.yml +++ b/roles/web-app-bluesky/meta/services.yml @@ -62,7 +62,7 @@ web: view: enabled: false repository: https://github.com/bluesky-social/social-app.git - ref: "1.134.0" + ref: "1.135.0" ports: local: http: 8051 @@ -75,7 +75,7 @@ view: pids_limit: 256 pds: image: ghcr.io/bluesky-social/pds - version: 0.4.0 + version: 0.4.66 min_storage: 290MB cpus: "1.0" mem_reservation: 1g diff --git a/roles/web-app-bluesky/templates/env.j2 b/roles/web-app-bluesky/templates/env.j2 index 8716c7d9c0..e0a1050b73 100644 --- a/roles/web-app-bluesky/templates/env.j2 +++ b/roles/web-app-bluesky/templates/env.j2 @@ -15,7 +15,6 @@ PDS_SERVICE_DID=did:web:{{ BLUESKY_API_DOMAIN }} {% set email = lookup('email', application_id) %} {% if email.enabled | bool %} -PDS_ADMIN_EMAIL={{ BLUESKY_ADMIN_EMAIL }} PDS_EMAIL_SMTP_URL=smtps://{{ email.username }}:{{ email.password }}@{{ email.host }}:{{ email.port }}/ PDS_EMAIL_FROM_ADDRESS={{ email['from'] }} {% endif %} diff --git a/roles/web-app-bluesky/vars/main.yml b/roles/web-app-bluesky/vars/main.yml index 187a57bc12..47cdad1b47 100644 --- a/roles/web-app-bluesky/vars/main.yml +++ b/roles/web-app-bluesky/vars/main.yml @@ -20,8 +20,8 @@ BLUESKY_WEB_IMAGE_CUSTOM: "bluesky_web_custom" BLUESKY_VIEW_ENABLED: "{{ lookup('config', application_id, 'services.view.enabled') }}" BLUESKY_VIEW_DOMAIN_LOCAL: "{{ lookup('domains', application_id).view }}" -BLUESKY_VIEW_DOMAIN_FINAL: "{{ BLUESKY_VIEW_DOMAIN_LOCAL if BLUESKY_VIEW_ENABLED else 'api.bsky.app' }}" -BLUESKY_VIEW_URL: "{{ lookup('tls', application_id, 'protocols.web') }}://{{ BLUESKY_VIEW_DOMAIN_FINAL }}" +BLUESKY_VIEW_DOMAIN_FINAL: "{{ BLUESKY_VIEW_DOMAIN_LOCAL if BLUESKY_VIEW_ENABLED | bool else 'api.bsky.app' }}" +BLUESKY_VIEW_URL: "{{ lookup('tls', application_id, 'protocols.web') if BLUESKY_VIEW_ENABLED | bool else 'https' }}://{{ BLUESKY_VIEW_DOMAIN_FINAL }}" BLUESKY_VIEW_DID: "did:web:{{ BLUESKY_VIEW_DOMAIN_FINAL }}" BLUESKY_VIEW_PORT: "{{ lookup('config', 'web-app-bluesky', 'services.view.ports.local.http') }}" diff --git a/roles/web-app-bridgy-fed/meta/services.yml b/roles/web-app-bridgy-fed/meta/services.yml index 175df25a2c..c47b937e41 100644 --- a/roles/web-app-bridgy-fed/meta/services.yml +++ b/roles/web-app-bridgy-fed/meta/services.yml @@ -48,6 +48,7 @@ bridgy-fed: swarm: enabled: true image: python + # nocheck: docker-version upstream's requirements.pinned.txt pins cbor2 6.1.4, which ships wheels up to cp314 and needs Rust >= 1.88 to build from source. TODO: return to 3.15-slim once the pinned upstream ref carries cbor2 >= 6.1.5 (cp315 wheels) version: 3.14-slim name: web-app-bridgy-fed repository: https://github.com/snarfed/bridgy-fed.git diff --git a/roles/web-app-erpnext/meta/services.yml b/roles/web-app-erpnext/meta/services.yml index cc24f369e8..aeadaa0b54 100644 --- a/roles/web-app-erpnext/meta/services.yml +++ b/roles/web-app-erpnext/meta/services.yml @@ -65,7 +65,7 @@ erpnext: enabled: true shared: true image: frappe/erpnext - version: "v16.50.0" + version: "v16.51.0" name: erpnext min_storage: 1017MB ports: diff --git a/roles/web-app-espocrm/meta/services.yml b/roles/web-app-espocrm/meta/services.yml index 59bbaf386e..fcde908d0c 100644 --- a/roles/web-app-espocrm/meta/services.yml +++ b/roles/web-app-espocrm/meta/services.yml @@ -37,7 +37,7 @@ espocrm: swarm: enabled: true image: espocrm/espocrm - version: "10.0.9" + version: "10.0.10" custom: true name: espocrm min_storage: 470MB diff --git a/roles/web-app-fediwall/meta/services.yml b/roles/web-app-fediwall/meta/services.yml index 91af39c490..ed2eb8fb31 100644 --- a/roles/web-app-fediwall/meta/services.yml +++ b/roles/web-app-fediwall/meta/services.yml @@ -52,7 +52,7 @@ fediwall: swarm: enabled: true image: nginx - version: 1.31.6-alpine + version: 1.31.7-alpine custom: true upstream: version: "1.4.0" diff --git a/roles/web-app-hugo/meta/services.yml b/roles/web-app-hugo/meta/services.yml index f2b70dadc4..98f3b15f8d 100644 --- a/roles/web-app-hugo/meta/services.yml +++ b/roles/web-app-hugo/meta/services.yml @@ -35,7 +35,7 @@ hugo: version: 1.31-alpine custom: true builder_image: ghcr.io/gohugoio/hugo - builder_version: v0.167.0 + builder_version: v0.168.0 source_repository: https://github.com/gohugoio/hugoDocs.git source_version: v0.165.0 update: diff --git a/roles/web-app-keycloak/meta/services.yml b/roles/web-app-keycloak/meta/services.yml index 2071d0fa64..379394b130 100644 --- a/roles/web-app-keycloak/meta/services.yml +++ b/roles/web-app-keycloak/meta/services.yml @@ -51,7 +51,7 @@ keycloak: shared: true provides: sso image: quay.io/keycloak/keycloak - version: "26.8.0" + version: "26.8.1" name: keycloak min_storage: 468MB cpus: "2.0" diff --git a/roles/web-app-littlejs/meta/services.yml b/roles/web-app-littlejs/meta/services.yml index b1b6376874..f940fb8081 100644 --- a/roles/web-app-littlejs/meta/services.yml +++ b/roles/web-app-littlejs/meta/services.yml @@ -38,7 +38,7 @@ littlejs: repository: https://github.com/KilledByAPixel/LittleJS.git ref: main image: nginx - version: 1.31.6-alpine + version: 1.31.7-alpine custom: true ports: local: diff --git a/roles/web-app-mailu/docs/Testing.md b/roles/web-app-mailu/docs/Testing.md index f5e69f4230..db0f20a84f 100644 --- a/roles/web-app-mailu/docs/Testing.md +++ b/roles/web-app-mailu/docs/Testing.md @@ -2,5 +2,5 @@ Use the following tools to test your server instance: -- [SSL-Tools Mailserver Test](https://de.ssl-tools.net/mailservers/) +- [Internet.nl Email Test](https://internet.nl/test-mail/) - [TestEmail.de](http://testemail.de/) diff --git a/roles/web-app-matrix/meta/services.yml b/roles/web-app-matrix/meta/services.yml index a4c905e212..2d8cf28134 100644 --- a/roles/web-app-matrix/meta/services.yml +++ b/roles/web-app-matrix/meta/services.yml @@ -43,7 +43,7 @@ postgres: synapse: healthcheck: flavor: curl - version: v1.162.0 + version: v1.163.0 image: matrixdotorg/synapse name: matrix-synapse backup: @@ -66,7 +66,7 @@ synapse: element: healthcheck: flavor: wget - version: v1.12.30 + version: v1.12.31 image: vectorim/element-web name: matrix-element ports: diff --git a/roles/web-app-mattermost/meta/services.yml b/roles/web-app-mattermost/meta/services.yml index 9055d0823d..eb42143a3e 100644 --- a/roles/web-app-mattermost/meta/services.yml +++ b/roles/web-app-mattermost/meta/services.yml @@ -46,7 +46,7 @@ mattermost: shared: true user_access_token_enabled: "{{ 'web-app-nextcloud' in group_names }}" image: mattermost/mattermost-team-edition - version: 11.11.1 + version: 12.0.0 min_storage: 659MB backup: no_stop_required: true diff --git a/roles/web-app-n8n/files/javascript/hooks.js b/roles/web-app-n8n/files/javascript/hooks.js index 80e3f06db6..0864ffa366 100644 --- a/roles/web-app-n8n/files/javascript/hooks.js +++ b/roles/web-app-n8n/files/javascript/hooks.js @@ -40,18 +40,20 @@ module.exports = { n8n: { ready: [ async function ({ app }, config) { - const headerName = process.env.N8N_FORWARD_AUTH_HEADER; - if (!headerName) { - this.logger?.info('N8N_FORWARD_AUTH_HEADER not set; SSO middleware disabled.'); - return; - } - - this.logger?.info(`SSO middleware initializing with header: ${headerName}`); - const { createRequire } = require('module'); const { dirname, resolve } = require('path'); const { randomBytes } = require('crypto'); const n8nRequire = createRequire(require.resolve('n8n')); + const logger = n8nRequire('@n8n/di').Container.get(n8nRequire('@n8n/backend-common').Logger); + + const headerName = process.env.N8N_FORWARD_AUTH_HEADER; + if (!headerName) { + logger.info('N8N_FORWARD_AUTH_HEADER not set; SSO middleware disabled.'); + return; + } + + logger.info(`SSO middleware initializing with header: ${headerName}`); + const Layer = createRequire(n8nRequire.resolve('express'))('router/lib/layer'); const { hash } = n8nRequire('bcryptjs'); const { issueCookie } = require(resolve(dirname(require.resolve('n8n')), 'auth/jwt')); @@ -92,16 +94,16 @@ module.exports = { const payload = JSON.parse(Buffer.from(parts[1], 'base64').toString()); firstName = payload.given_name || ''; lastName = payload.family_name || ''; - this.logger?.debug(`Extracted from access token: firstName="${firstName}", lastName="${lastName}"`); + logger.debug(`Extracted from access token: firstName="${firstName}", lastName="${lastName}"`); } } catch (e) { - this.logger?.debug(`Failed to decode access token: ${e.message}`); + logger.debug(`Failed to decode access token: ${e.message}`); } } // If the forward-auth header with email is missing — do nothing if (!emailHeader) { - this.logger?.debug(`No ${headerName} header found, skipping SSO auto-login`); + logger.debug(`No ${headerName} header found, skipping SSO auto-login`); return next(); } @@ -110,11 +112,11 @@ module.exports = { const userLastName = Array.isArray(lastName) ? lastName[0] : String(lastName).trim(); if (!userEmail) { - this.logger?.debug(`Empty ${headerName} header, skipping SSO auto-login`); + logger.debug(`Empty ${headerName} header, skipping SSO auto-login`); return next(); } - this.logger?.info(`SSO auto-login attempt for email: ${userEmail}`); + logger.info(`SSO auto-login attempt for email: ${userEmail}`); let user = await UserRepo.findOne({ where: { email: userEmail }, @@ -135,7 +137,7 @@ module.exports = { const created = await UserRepo.createUserWithProject(userData); user = created.user; - this.logger?.info(`Created new user: ${userEmail} (${userFirstName} ${userLastName}) via SSO`); + logger.info(`Created new user: ${userEmail} (${userFirstName} ${userLastName}) via SSO`); } else { let changed = false; if (userFirstName && user.firstName !== userFirstName) { @@ -148,14 +150,14 @@ module.exports = { } if (changed) { await UserRepo.save(user); - this.logger?.info(`Updated user: ${userEmail} (${userFirstName} ${userLastName}) via SSO`); + logger.info(`Updated user: ${userEmail} (${userFirstName} ${userLastName}) via SSO`); } else { - this.logger?.info(`Existing user logged in: ${userEmail} via SSO`); + logger.info(`Existing user logged in: ${userEmail} via SSO`); } } if (!user.role) { - this.logger?.error(`User ${userEmail} has no valid role; cannot issue cookie.`); + logger.error(`User ${userEmail} has no valid role; cannot issue cookie.`); res.statusCode = 401; res.end(`User ${userEmail} has no valid role. Ask admin to assign a role.`); return; @@ -168,14 +170,14 @@ module.exports = { return next(); } catch (error) { - this.logger?.error(`SSO middleware error: ${error.message}`); + logger.error(`SSO middleware error: ${error.message}`); return next(error); } }); // Insert our middleware right after cookieParser stack.splice(idx + 1, 0, layer); - this.logger?.info('SSO middleware initialized successfully'); + logger.info('SSO middleware initialized successfully'); } ] } diff --git a/roles/web-app-n8n/files/playwright/test-litellm-gateway.js b/roles/web-app-n8n/files/playwright/test-litellm-gateway.js index 97d55b4d57..7a77fefaed 100644 --- a/roles/web-app-n8n/files/playwright/test-litellm-gateway.js +++ b/roles/web-app-n8n/files/playwright/test-litellm-gateway.js @@ -8,6 +8,7 @@ const gatewayModel = decodeDotenvQuotedValue(process.env.N8N_LITELLM_MODEL); const CREDENTIAL_TYPE = "openAiApi"; const PROBE_WORKFLOW = "infinito:litellm-probe"; +const TRIGGER_NODE = "Start"; const REQUEST_NODE = "Gateway"; const PROMPT = "Reply with exactly the word: pong"; const FINAL_STATUSES = new Set(["success", "error", "crashed", "canceled"]); @@ -158,7 +159,7 @@ function probeWorkflow(credentialId, endpoint) { nodes: [ { id: "infinito-litellm-start", - name: "Start", + name: TRIGGER_NODE, type: "n8n-nodes-base.manualTrigger", typeVersion: 1, position: [0, 0], @@ -188,7 +189,7 @@ function probeWorkflow(credentialId, endpoint) { credentials: { [CREDENTIAL_TYPE]: { id: credentialId, name: credentialName } }, }, ], - connections: { Start: { main: [[{ node: REQUEST_NODE, type: "main", index: 0 }]] } }, + connections: { [TRIGGER_NODE]: { main: [[{ node: REQUEST_NODE, type: "main", index: 0 }]] } }, }; } @@ -286,7 +287,7 @@ exports.register = function (shared) { try { const run = await rest("POST", `/rest/workflows/${workflowId}/run`, { - workflowData: { ...workflow, id: workflowId }, + triggerToStartFrom: { name: TRIGGER_NODE }, }); expect( run.status, diff --git a/roles/web-app-n8n/files/python/provision/client.py b/roles/web-app-n8n/files/python/provision/client.py index e1c6241d3d..e81b3598cb 100644 --- a/roles/web-app-n8n/files/python/provision/client.py +++ b/roles/web-app-n8n/files/python/provision/client.py @@ -25,6 +25,7 @@ import os import re import sys import urllib.error +import urllib.parse import urllib.request BASE = os.environ.get("N8N_BASE", "").rstrip("/") @@ -38,6 +39,7 @@ PROVIDERS = json.loads(os.environ.get("N8N_MCP_PROVIDERS", "[]")) REST = "/rest" PUBLIC = "/api/v1" +PAGE = 250 CREDENTIAL_TYPE = "httpBearerAuth" BEARER_AUTH = "bearerAuth" CLIENT_PREFIX = "infinito:mcp-client:" @@ -90,6 +92,103 @@ def call(path, method="GET", payload=None, api_key=None): return error.code, error.read().decode(errors="replace") +def shape(value): + """Describe a value by its type and, for a mapping, its keys, never its content. + + Args: + value: decoded response fragment. + """ + if isinstance(value, dict): + return f"a mapping with keys {sorted(map(str, value))}" + if isinstance(value, list): + return f"a list of {sorted({type(item).__name__ for item in value})}" + return f"a {type(value).__name__}" + + +def records(body, what): + """Return the records of a list response. + + Args: + body: decoded body: ``{"data": [...]}``, a bare list, or the paged + ``{"data": {"items": [...], ...}}`` of n8n 2. + what: noun naming the collection, for the failure message. + """ + listed = body.get("data", body) if isinstance(body, dict) else body + if isinstance(listed, dict) and "items" in listed: + listed = listed["items"] + if listed is None: + return [] + if not isinstance(listed, list) or not all( + isinstance(item, dict) for item in listed + ): + sys.exit(f"FAILED listing {what}: expected records, got {shape(listed)}") + return listed + + +def record(body, what): + """Return the single record of a create response. + + Args: + body: decoded body, either ``{"data": {...}}`` or a bare mapping. + what: noun naming the record, for the failure message. + """ + created = body.get("data", body) if isinstance(body, dict) else body + if not isinstance(created, dict): + sys.exit(f"FAILED creating {what}: expected a record, got {shape(created)}") + return created + + +def api_keys_labelled(label): + """Return every API key whose label n8n matches against ``label``, all pages. + + Args: + label: text n8n looks for inside each label. + """ + found = [] + seen = set() + while True: + query = urllib.parse.urlencode( + {"label": label, "take": PAGE, "skip": len(found)} + ) + status, body = call(f"{REST}/api-keys?{query}") + if status != 200: + sys.exit(f"FAILED listing api keys: {status} {body}") + page = records(body, "api keys") + ids = {str(key.get("id")) for key in page} + if page and ids <= seen: + sys.exit(f"FAILED listing api keys: page at skip {len(found)} repeats") + seen |= ids + found.extend(page) + if len(page) < PAGE: + return found + + +def workflows_named(key, name): + """Return every workflow n8n matches against ``name``, following ``nextCursor``. + + Args: + key: the public-API key. + name: workflow name n8n filters on. + """ + found = [] + cursors = set() + query = {"name": name, "limit": PAGE} + while True: + status, body = call( + f"{PUBLIC}/workflows?{urllib.parse.urlencode(query)}", api_key=key + ) + if status != 200: + sys.exit(f"FAILED listing workflows: {status} {body}") + found.extend(records(body, "workflows")) + cursor = body.get("nextCursor") if isinstance(body, dict) else None + if not cursor: + return found + if cursor in cursors: + sys.exit("FAILED listing workflows: nextCursor repeats") + cursors.add(cursor) + query = {"name": name, "cursor": cursor} + + def login(): status, body = call( f"{REST}/login", @@ -106,14 +205,12 @@ def api_key(): n8n returns the usable secret only once, as ``rawApiKey``, and a listed key is redacted, so the key cannot be carried between runs. It is minted here and revoked before the run ends; an entry left behind under the managed name - belongs to a run that died and is deleted rather than guessed at. + belongs to a run that died and is deleted rather than guessed at. n8n matches + ``label`` as a substring, so the exact match happens here. """ - status, body = call(f"{REST}/api-keys") - if status != 200: - sys.exit(f"FAILED listing api keys: {status} {body}") - - existing = (body or {}).get("data") if isinstance(body, dict) else body - matches = [key for key in existing or [] if key.get("label") == KEY_NAME] + matches = [ + key for key in api_keys_labelled(KEY_NAME) if key.get("label") == KEY_NAME + ] if len(matches) > 1: sys.exit(f"FAILED: {len(matches)} api keys named {KEY_NAME}") for key in matches: @@ -136,7 +233,7 @@ def api_key(): ) if status not in (200, 201): sys.exit(f"FAILED creating api key {KEY_NAME}: {status} {body}") - created = (body or {}).get("data") if isinstance(body, dict) else body + created = record(body, f"api key {KEY_NAME}") return str(created["rawApiKey"]), str(created["id"]) @@ -174,8 +271,7 @@ def listed_credentials(): status, body = call(f"{REST}/credentials") if status != 200: sys.exit(f"FAILED listing credentials: {status} {body}") - existing = (body or {}).get("data") if isinstance(body, dict) else body - return list(existing or []) + return records(body, "credentials") def credential(provider): @@ -198,7 +294,7 @@ def credential(provider): status, body = call(f"{REST}/credentials", method="POST", payload=payload) if status not in (200, 201): sys.exit(f"FAILED creating {name}: {status} {body}") - created = (body or {}).get("data") if isinstance(body, dict) else body + created = record(body, name) return {"id": str(created["id"]), "name": name}, True status, body = call( @@ -314,11 +410,11 @@ def managed_workflow(key): Args: key: the public-API key. """ - status, body = call(f"{PUBLIC}/workflows", api_key=key) - if status != 200: - sys.exit(f"FAILED listing workflows: {status} {body}") - existing = (body or {}).get("data") if isinstance(body, dict) else body - matches = [flow for flow in existing or [] if flow.get("name") == WORKFLOW_NAME] + matches = [ + flow + for flow in workflows_named(key, WORKFLOW_NAME) + if flow.get("name") == WORKFLOW_NAME + ] if len(matches) > 1: sys.exit(f"FAILED: {len(matches)} workflows named {WORKFLOW_NAME}") return matches[0] if matches else None diff --git a/roles/web-app-n8n/files/python/provision/mcp.py b/roles/web-app-n8n/files/python/provision/mcp.py index e54f5f5d39..81d485c19c 100644 --- a/roles/web-app-n8n/files/python/provision/mcp.py +++ b/roles/web-app-n8n/files/python/provision/mcp.py @@ -19,6 +19,7 @@ import json import os import sys import urllib.error +import urllib.parse import urllib.request BASE = os.environ.get("N8N_BASE", "").rstrip("/") @@ -32,6 +33,7 @@ CREDENTIAL_NAME = os.environ.get("N8N_MCP_CREDENTIAL", "") REST = "/rest" PUBLIC = "/api/v1" +PAGE = 250 TRIGGER_TYPE = "@n8n/n8n-nodes-langchain.mcpTrigger" TRIGGER_VERSION = 2 TRIGGER_NAME = "Infinito MCP Trigger" @@ -95,6 +97,103 @@ def call(path, method="GET", payload=None, api_key=None): return error.code, error.read().decode(errors="replace") +def shape(value): + """Describe a value by its type and, for a mapping, its keys, never its content. + + Args: + value: decoded response fragment. + """ + if isinstance(value, dict): + return f"a mapping with keys {sorted(map(str, value))}" + if isinstance(value, list): + return f"a list of {sorted({type(item).__name__ for item in value})}" + return f"a {type(value).__name__}" + + +def records(body, what): + """Return the records of a list response. + + Args: + body: decoded body: ``{"data": [...]}``, a bare list, or the paged + ``{"data": {"items": [...], ...}}`` of n8n 2. + what: noun naming the collection, for the failure message. + """ + listed = body.get("data", body) if isinstance(body, dict) else body + if isinstance(listed, dict) and "items" in listed: + listed = listed["items"] + if listed is None: + return [] + if not isinstance(listed, list) or not all( + isinstance(item, dict) for item in listed + ): + sys.exit(f"FAILED listing {what}: expected records, got {shape(listed)}") + return listed + + +def record(body, what): + """Return the single record of a create response. + + Args: + body: decoded body, either ``{"data": {...}}`` or a bare mapping. + what: noun naming the record, for the failure message. + """ + created = body.get("data", body) if isinstance(body, dict) else body + if not isinstance(created, dict): + sys.exit(f"FAILED creating {what}: expected a record, got {shape(created)}") + return created + + +def api_keys_labelled(label): + """Return every API key whose label n8n matches against ``label``, all pages. + + Args: + label: text n8n looks for inside each label. + """ + found = [] + seen = set() + while True: + query = urllib.parse.urlencode( + {"label": label, "take": PAGE, "skip": len(found)} + ) + status, body = call(f"{REST}/api-keys?{query}") + if status != 200: + sys.exit(f"FAILED listing api keys: {status} {body}") + page = records(body, "api keys") + ids = {str(key.get("id")) for key in page} + if page and ids <= seen: + sys.exit(f"FAILED listing api keys: page at skip {len(found)} repeats") + seen |= ids + found.extend(page) + if len(page) < PAGE: + return found + + +def workflows_named(key, name): + """Return every workflow n8n matches against ``name``, following ``nextCursor``. + + Args: + key: the public-API key. + name: workflow name n8n filters on. + """ + found = [] + cursors = set() + query = {"name": name, "limit": PAGE} + while True: + status, body = call( + f"{PUBLIC}/workflows?{urllib.parse.urlencode(query)}", api_key=key + ) + if status != 200: + sys.exit(f"FAILED listing workflows: {status} {body}") + found.extend(records(body, "workflows")) + cursor = body.get("nextCursor") if isinstance(body, dict) else None + if not cursor: + return found + if cursor in cursors: + sys.exit("FAILED listing workflows: nextCursor repeats") + cursors.add(cursor) + query = {"name": name, "cursor": cursor} + + def login(): status, body = call( f"{REST}/login", @@ -111,14 +210,12 @@ def api_key(): n8n returns the usable secret only once, as ``rawApiKey``, and a listed key is redacted, so the key cannot be carried between runs. It is minted here and revoked before the run ends; an entry left behind under the managed name - belongs to a run that died and is deleted rather than guessed at. + belongs to a run that died and is deleted rather than guessed at. n8n matches + ``label`` as a substring, so the exact match happens here. """ - status, body = call(f"{REST}/api-keys") - if status != 200: - sys.exit(f"FAILED listing api keys: {status} {body}") - - existing = (body or {}).get("data") if isinstance(body, dict) else body - matches = [key for key in existing or [] if key.get("label") == KEY_NAME] + matches = [ + key for key in api_keys_labelled(KEY_NAME) if key.get("label") == KEY_NAME + ] if len(matches) > 1: sys.exit(f"FAILED: {len(matches)} api keys named {KEY_NAME}") for key in matches: @@ -134,13 +231,14 @@ def api_key(): "workflow:read", "workflow:update", "workflow:list", + "workflow:activate", ], "expiresAt": None, }, ) if status not in (200, 201): sys.exit(f"FAILED creating api key {KEY_NAME}: {status} {body}") - created = (body or {}).get("data") if isinstance(body, dict) else body + created = record(body, f"api key {KEY_NAME}") return str(created["rawApiKey"]), str(created["id"]) @@ -166,8 +264,11 @@ def bearer_credential(): if status != 200: sys.exit(f"FAILED listing credentials: {status} {body}") - existing = (body or {}).get("data") if isinstance(body, dict) else body - matches = [item for item in existing or [] if item.get("name") == CREDENTIAL_NAME] + matches = [ + item + for item in records(body, "credentials") + if item.get("name") == CREDENTIAL_NAME + ] if len(matches) > 1: sys.exit(f"FAILED: {len(matches)} credentials named {CREDENTIAL_NAME}") @@ -180,7 +281,7 @@ def bearer_credential(): status, body = call(f"{REST}/credentials", method="POST", payload=payload) if status not in (200, 201): sys.exit(f"FAILED creating {CREDENTIAL_NAME}: {status} {body}") - created = (body or {}).get("data") if isinstance(body, dict) else body + created = record(body, CREDENTIAL_NAME) return {"id": str(created["id"]), "name": CREDENTIAL_NAME}, True status, body = call( @@ -264,12 +365,11 @@ def upsert_workflow(key, credential): key: the public-API key. credential: ``{"id", "name"}`` reference to the managed bearer credential. """ - status, body = call(f"{PUBLIC}/workflows", api_key=key) - if status != 200: - sys.exit(f"FAILED listing workflows: {status} {body}") - - existing = (body or {}).get("data") if isinstance(body, dict) else body - matches = [flow for flow in existing or [] if flow.get("name") == WORKFLOW_NAME] + matches = [ + flow + for flow in workflows_named(key, WORKFLOW_NAME) + if flow.get("name") == WORKFLOW_NAME + ] if len(matches) > 1: sys.exit(f"FAILED: {len(matches)} workflows named {WORKFLOW_NAME}") diff --git a/roles/web-app-n8n/meta/services.yml b/roles/web-app-n8n/meta/services.yml index 187bfe2b78..59cd03d88d 100644 --- a/roles/web-app-n8n/meta/services.yml +++ b/roles/web-app-n8n/meta/services.yml @@ -67,7 +67,7 @@ n8n: backup: no_stop_required: false image: docker.n8n.io/n8nio/n8n - version: "2.43.4" + version: "2.44.2" update: type: registry_tags image: docker.n8n.io/n8nio/n8n diff --git a/roles/web-app-opentalk/meta/services.yml b/roles/web-app-opentalk/meta/services.yml index 4ceba7a5ba..8640838ba9 100644 --- a/roles/web-app-opentalk/meta/services.yml +++ b/roles/web-app-opentalk/meta/services.yml @@ -104,7 +104,7 @@ controller: livekit: name: opentalk-livekit image: livekit/livekit-server - version: v1.13.9 + version: v1.13.10 min_storage: 238MB cpus: "2.0" mem_reservation: 512m diff --git a/roles/web-app-prometheus/templates/configuration/blackbox.yml.j2 b/roles/web-app-prometheus/templates/configuration/blackbox.yml.j2 index 48147594e0..58feb765be 100644 --- a/roles/web-app-prometheus/templates/configuration/blackbox.yml.j2 +++ b/roles/web-app-prometheus/templates/configuration/blackbox.yml.j2 @@ -26,5 +26,19 @@ modules: fail_if_ssl: false fail_if_not_ssl: false tls_config: - insecure_skip_verify: {{ ((TLS_MODE | lower) == 'self_signed' and not lookup('ca_injected', application_id)) | lower }} + insecure_skip_verify: false + preferred_ip_protocol: "ip4" + + http_ready_private_ca: + prober: http + timeout: 10s + http: + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + valid_status_codes: [200, 301, 302, 303, 307, 308, 401, 403] + method: GET + follow_redirects: false + fail_if_ssl: false + fail_if_not_ssl: false + tls_config: + insecure_skip_verify: true preferred_ip_protocol: "ip4" diff --git a/roles/web-app-prometheus/templates/configuration/prometheus.yml.j2 b/roles/web-app-prometheus/templates/configuration/prometheus.yml.j2 index 86ea8e71aa..3292b05829 100644 --- a/roles/web-app-prometheus/templates/configuration/prometheus.yml.j2 +++ b/roles/web-app-prometheus/templates/configuration/prometheus.yml.j2 @@ -43,6 +43,7 @@ scrape_configs: {% set canonical_map = lookup('domains') %} {% set probed_roles = lookup('roles_with_service', 'prometheus') | map(attribute='id') | list %} +{% set private_ca_unverifiable = not lookup('ca_injected', application_id) %} - job_name: "blackbox-healthz" metrics_path: /probe params: @@ -59,6 +60,9 @@ scrape_configs: {% endfor %} labels: app: "{{ _application_id }}" +{% if private_ca_unverifiable and lookup('tls', _application_id, 'mode') == 'self_signed' %} + __param_module: http_ready_private_ca +{% endif %} {% endif %} {% endif %} {% endfor %} diff --git a/roles/web-app-seaweedfs/meta/services.yml b/roles/web-app-seaweedfs/meta/services.yml index 2ba8b58900..c99073bd47 100644 --- a/roles/web-app-seaweedfs/meta/services.yml +++ b/roles/web-app-seaweedfs/meta/services.yml @@ -12,6 +12,8 @@ frontend: domains: - filer - master +# nocheck: variants-coverage the onion gate is a hard invariant; no variant may force this consumer on a tor node. +# nocheck: bond-baseline same reason: variant 0 must not pin this consumer on. sso: bond: 1 enabled: "{{ ('web-app-keycloak' in group_names) and ('svc-net-tor' not in group_names) }}" diff --git a/roles/web-app-seaweedfs/meta/variants.yml b/roles/web-app-seaweedfs/meta/variants.yml index 678afaac86..4ea8d2cbef 100644 --- a/roles/web-app-seaweedfs/meta/variants.yml +++ b/roles/web-app-seaweedfs/meta/variants.yml @@ -6,8 +6,8 @@ container_backup: enabled: true sso: - enabled: true - shared: true + enabled: "{{ 'svc-net-tor' not in group_names }}" + shared: "{{ 'svc-net-tor' not in group_names }}" logout: enabled: "{{ 'svc-net-tor' not in group_names }}" shared: "{{ 'svc-net-tor' not in group_names }}" diff --git a/roles/web-app-socialhome/meta/services.yml b/roles/web-app-socialhome/meta/services.yml index d3ff755d73..d7f917433f 100644 --- a/roles/web-app-socialhome/meta/services.yml +++ b/roles/web-app-socialhome/meta/services.yml @@ -13,7 +13,7 @@ socialhome: # nocheck: default-placement-manager embedded sqlite in a node-local volume; no clustered storage placement: manager image: ghcr.io/social-home-io/socialhome - version: "2026.10.11" + version: "2026.10.14" ports: local: http: 8089 diff --git a/roles/web-app-stalwart/README.md b/roles/web-app-stalwart/README.md index 83d455f38a..e1c2cc16a0 100644 --- a/roles/web-app-stalwart/README.md +++ b/roles/web-app-stalwart/README.md @@ -57,7 +57,7 @@ flowchart LR STALWART -- "OIDC directory
(token validation)" --> KC MAIL_VHOST -. "SPA login: PKCE
(stalwart-webui client)" .-> KC WEBMAIL_VHOST -. "OAuth login" .-> KC - APPS -- "STARTTLS :25, no AUTH
(SSO relay, trusted networks)" --> STALWART + APPS -- "SMTPS :465, AUTH
(bot credential from the token store)" --> STALWART WORLD <-- "public mail ports
(only when active MAIL_PROVIDER)" --> STALWART ``` @@ -151,19 +151,28 @@ delegates **interactive** authentication to Keycloak: **Design constraint (validated against the live JMAP schema):** Stalwart's authentication directory is *Internal XOR one external directory* — there is no -chaining or fallback. Enabling SSO therefore **disables password submission** -for every account, including the machine `no-reply` account. To keep outbound -notifications working, the role: +chaining or fallback. Enabling SSO therefore **disables the account password** +for every account, including the machine `no-reply` account. App passwords are +checked before the directory and keep working, so with SSO enabled the role: -1. Widens `x:MtaStageRcpt.allowRelaying` to trust the internal Docker networks - (`STALWART_TRUSTED_NETWORKS`), so the bot relays without SMTP AUTH; and -2. Self-declares `services.sso.oidc.submission_via_relay: true`, which makes - [`plugins/lookup/email.py`](../../plugins/lookup/email.py) switch the - `no-reply` client to unauthenticated STARTTLS relay on port 25. +1. Creates one app password per bot account, acting for that account as the + recovery admin (`tasks/04_manage_user/app_password.yml`), and +2. Stores it in the token store, where + [`plugins/lookup/email.py`](../../plugins/lookup/email.py) reads the SMTP + password of the `no-reply` client. -With SSO disabled the role uses the Internal directory and password submission, -exactly as before. Mailu keeps password submission in both modes and does not -set `submission_via_relay`. +A bot gets a new app password when its entry named +`STALWART_BOT_APP_PASSWORD_DESCRIPTION` is deleted in Stalwart or its token is +removed from the token store. The app password it replaces is renamed to +`STALWART_BOT_APP_PASSWORD_RETIRED_DESCRIPTION` and stays valid until the +service-loader post-load flush drops it +(`tasks/04_manage_user/app_password_drop.yml`), or until right after the new +one is stored when the role runs outside the service loader. A deploy that +stops before the flush leaves it to the next deploy. Relaying stays limited to +authenticated sessions in both modes (`STALWART_RELAY_RULE`). + +With SSO disabled the role uses the Internal directory and the bot's derived +account password. ## Calendar & Contacts (CalDAV / CardDAV / WebDAV) @@ -195,8 +204,15 @@ account (or the Keycloak SSO token when SSO is enabled). - **`config.json` is bootstrap-only** (data store selection); all other configuration lives in the database and is provisioned over JMAP (`templates/jmap/*.json.j2`). -- **Under SSO, trusted internal networks relay without SMTP AUTH** — the - no-reply bot has no password once the auth directory is OIDC. +- **Only authenticated sessions relay, with or without SSO.** Under SSO a bot + authenticates with an app password. +- **Address bans expire after `services.stalwart.ban_period_seconds`** (one + hour by default). Without a period Stalwart keeps a ban forever, and the + reverse proxy, the published ports, Tor and the swarm ingress hand many + clients one address. +- **Bans and failed logins also go to the container output** + (`STALWART_BAN_TRACER`, `tasks/tracing.yml`), next to the daily log file in + the data volume. ## Features diff --git a/roles/web-app-stalwart/files/test/probe_mail.py b/roles/web-app-stalwart/files/test/probe_mail.py index b6210e7f4d..11b280b121 100755 --- a/roles/web-app-stalwart/files/test/probe_mail.py +++ b/roles/web-app-stalwart/files/test/probe_mail.py @@ -10,7 +10,10 @@ speaks IMAP through imaplib. probe_mail.py send probe_mail.py find + probe_mail.py relay +`relay` exits 0 when an unauthenticated session is refused an external recipient +and the same recipient is accepted once `` has authenticated, 1 otherwise. `find` exits 0 when the subject is present in INBOX or Junk, 1 when it is not. Both mailboxes are searched because a test domain publishes no mail-auth DNS, so a correctly delivered message is legitimately filed under Junk. Certificate @@ -32,10 +35,20 @@ import sys from pathlib import Path SMTP_PORT = 25 +SMTPS_PORT = 465 IMAPS_PORT = 993 TIMEOUT = 60 MAILBOXES = ("INBOX", "Junk Mail") +EXTERNAL_RECIPIENT = "relay-probe@example.org" +RELAY_DENIED = 550 + + +def unverified_context() -> ssl.SSLContext: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + return context def send(host: str, sender: str, recipient: str, eml_path: str) -> int: @@ -45,12 +58,46 @@ def send(host: str, sender: str, recipient: str, eml_path: str) -> int: return 0 -def find(host: str, user: str, password: str, subject: str) -> int: - context = ssl.create_default_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE +def relay(host: str, sender: str, password: str) -> int: + with smtplib.SMTP(host, SMTP_PORT, timeout=TIMEOUT) as smtp: + smtp.ehlo() + smtp.mail(sender) + anonymous, _ = smtp.rcpt(EXTERNAL_RECIPIENT) + if anonymous != RELAY_DENIED: + print( + f"relay not denied: an unauthenticated session got {anonymous} " + f"for {EXTERNAL_RECIPIENT}, expected {RELAY_DENIED}", + file=sys.stderr, + ) + return 1 - imap = imaplib.IMAP4_SSL(host, IMAPS_PORT, ssl_context=context, timeout=TIMEOUT) + with smtplib.SMTP_SSL( + host, SMTPS_PORT, timeout=TIMEOUT, context=unverified_context() + ) as smtp: + try: + smtp.login(sender, password) + except smtplib.SMTPAuthenticationError as error: + print( + f"submission refused: {sender} cannot authenticate ({error.smtp_code})", + file=sys.stderr, + ) + return 1 + smtp.mail(sender) + authenticated, _ = smtp.rcpt(EXTERNAL_RECIPIENT) + if authenticated != 250: + print( + f"submission refused: {sender} authenticated but got {authenticated} " + f"for {EXTERNAL_RECIPIENT}", + file=sys.stderr, + ) + return 1 + return 0 + + +def find(host: str, user: str, password: str, subject: str) -> int: + imap = imaplib.IMAP4_SSL( + host, IMAPS_PORT, ssl_context=unverified_context(), timeout=TIMEOUT + ) imap._encoding = "utf-8" try: imap.login(user, password) @@ -76,6 +123,8 @@ def main(argv: list[str]) -> int: return send(*args) if action == "find" and len(args) == 4: return find(*args) + if action == "relay" and len(args) == 3: + return relay(*args) print(__doc__, file=sys.stderr) return 2 diff --git a/roles/web-app-stalwart/files/test/relay.sh b/roles/web-app-stalwart/files/test/relay.sh new file mode 100644 index 0000000000..1c5e980814 --- /dev/null +++ b/roles/web-app-stalwart/files/test/relay.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +: "${PYTHON_BIN:?missing PYTHON_BIN}" +: "${MAIL_HOST:?missing MAIL_HOST}" +: "${NO_REPLY_EMAIL:?missing NO_REPLY_EMAIL}" +: "${NO_REPLY_SMTP_PASSWORD:?missing NO_REPLY_SMTP_PASSWORD}" + +"${PYTHON_BIN}" "${SCRIPT_DIR}/probe_mail.py" relay \ + "${MAIL_HOST}" "${NO_REPLY_EMAIL}" "${NO_REPLY_SMTP_PASSWORD}" +echo "[relay] unauthenticated relay refused, ${NO_REPLY_EMAIL} submits authenticated" diff --git a/roles/web-app-stalwart/files/test/test.sh b/roles/web-app-stalwart/files/test/test.sh index 3b5d6f53de..adf4d4ee51 100755 --- a/roles/web-app-stalwart/files/test/test.sh +++ b/roles/web-app-stalwart/files/test/test.sh @@ -4,4 +4,5 @@ set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +bash "${SCRIPT_DIR}/relay.sh" bash "${SCRIPT_DIR}/verify_migration.sh" diff --git a/roles/web-app-stalwart/meta/services.yml b/roles/web-app-stalwart/meta/services.yml index fcb805d964..55d80b675f 100644 --- a/roles/web-app-stalwart/meta/services.yml +++ b/roles/web-app-stalwart/meta/services.yml @@ -8,7 +8,6 @@ sso: oidc: email_by_username: true enable_user_creation: true - submission_via_relay: true clients: webui: stalwart-webui webmail: stalwart-webmail @@ -54,6 +53,7 @@ stalwart: project_hard_restart: true migration: import_mailu: false + ban_period_seconds: 3600 # nocheck: min-storage-headroom Reason: the mail store (maildir blobs, FTS index, queue) lives in this service's volumes and dwarfs the image transfer size min_storage: 20GB domain: "{{ lookup('email').domain }}" diff --git a/roles/web-app-stalwart/tasks/04_manage_user.yml b/roles/web-app-stalwart/tasks/04_manage_user.yml index 2cd75d2f1a..b9af38c469 100644 --- a/roles/web-app-stalwart/tasks/04_manage_user.yml +++ b/roles/web-app-stalwart/tasks/04_manage_user.yml @@ -46,4 +46,14 @@ sys_token_store_user_key: "{{ stalwart_user_key }}" sys_token_store_app: "{{ application_id }}" sys_token_store_token: "{{ stalwart_user_secret }}" - when: "'bot' in stalwart_user_roles" + when: + - "'bot' in stalwart_user_roles" + - not (STALWART_SSO_ENABLED | bool) + +- name: "🔑 Keep an app password for bot '{{ stalwart_user_name }}' under SSO" + include_tasks: 04_manage_user/app_password.yml + vars: + stalwart_jmap_act_as: "{{ stalwart_user_name }}@{{ STALWART_DOMAIN }}" + when: + - "'bot' in stalwart_user_roles" + - STALWART_SSO_ENABLED | bool diff --git a/roles/web-app-stalwart/tasks/04_manage_user/app_password.yml b/roles/web-app-stalwart/tasks/04_manage_user/app_password.yml new file mode 100644 index 0000000000..41b61f5b45 --- /dev/null +++ b/roles/web-app-stalwart/tasks/04_manage_user/app_password.yml @@ -0,0 +1,74 @@ +--- +# Param: stalwart_jmap_act_as - address of the bot account, every call below runs as that account. +- name: "📖 List the app passwords of '{{ stalwart_user_name }}'" + include_tasks: "{{ role_path }}/tasks/jmap_call.yml" + vars: + stalwart_jmap_body: >- + {{ {'using': ['urn:ietf:params:jmap:core', 'urn:stalwart:jmap'], + 'methodCalls': [['x:AppPassword/get', {'ids': none}, 'c0']]} | to_json }} + +- name: "🔍 Decide whether '{{ stalwart_user_name }}' needs a new app password" + ansible.builtin.set_fact: + stalwart_app_password_ids: "{{ stalwart_app_password_owned }}" + stalwart_app_password_mint: "{{ stalwart_app_password_renew }}" + stalwart_app_password_drop: "{{ stalwart_app_password_retired + (stalwart_app_password_owned if stalwart_app_password_renew | bool else []) }}" + vars: + stalwart_app_password_owned: >- + {{ stalwart_jmap.methodResponses[0][1].list + | selectattr('description', 'equalto', STALWART_BOT_APP_PASSWORD_DESCRIPTION) + | map(attribute='id') | list }} + stalwart_app_password_retired: >- + {{ stalwart_jmap.methodResponses[0][1].list + | selectattr('description', 'equalto', STALWART_BOT_APP_PASSWORD_RETIRED_DESCRIPTION) + | map(attribute='id') | list }} + stalwart_app_password_stored: "{{ lookup('users', stalwart_user_key, {}).get('tokens', {}).get(application_id, '') }}" + stalwart_app_password_renew: >- + {{ (stalwart_app_password_owned | length != 1) + or (stalwart_app_password_stored | length == 0) + or (stalwart_app_password_stored == stalwart_user_secret) }} + +- name: "🔑 Replace the app password of '{{ stalwart_user_name }}'" + when: stalwart_app_password_mint | bool + block: + - name: "🔑 Mint the app password of '{{ stalwart_user_name }}'" + include_tasks: "{{ role_path }}/tasks/jmap_call.yml" + vars: + stalwart_jmap_body: >- + {{ {'using': ['urn:ietf:params:jmap:core', 'urn:stalwart:jmap'], + 'methodCalls': [['x:AppPassword/set', {'create': {'p': { + 'description': STALWART_BOT_APP_PASSWORD_DESCRIPTION + }}}, 'c0']]} | to_json }} + + - name: "💾 Persist the app password of '{{ stalwart_user_name }}'" + ansible.builtin.include_role: + name: sys-token-store + tasks_from: write.yml + vars: + sys_token_store_user_key: "{{ stalwart_user_key }}" + sys_token_store_app: "{{ application_id }}" + sys_token_store_token: "{{ stalwart_jmap.methodResponses[0][1].created.p.secret }}" + + - name: "🏷️ Retire the replaced app passwords of '{{ stalwart_user_name }}'" + include_tasks: "{{ role_path }}/tasks/jmap_call.yml" + vars: + stalwart_jmap_body: >- + {{ {'using': ['urn:ietf:params:jmap:core', 'urn:stalwart:jmap'], + 'methodCalls': [['x:AppPassword/set', {'update': dict(stalwart_app_password_ids + | zip([{'description': STALWART_BOT_APP_PASSWORD_RETIRED_DESCRIPTION}] * (stalwart_app_password_ids | length)))}, + 'c0']]} | to_json }} + when: stalwart_app_password_ids | length > 0 + +- name: "📥 Defer dropping the retired app passwords of '{{ stalwart_user_name }}'" + when: stalwart_app_password_drop | length > 0 + block: + - name: "📥 Record the retired app passwords of '{{ stalwart_user_name }}'" + ansible.builtin.set_fact: + stalwart_app_password_stale: "{{ stalwart_app_password_stale | combine({stalwart_jmap_act_as: stalwart_app_password_drop}) }}" + + - name: "📥 Queue the drop for the service-loader post-load flush" + include_role: + name: sys-service-loader + tasks_from: list_or_shoot.yml + vars: + list_or_shoot_roles: + - web-app-stalwart diff --git a/roles/web-app-stalwart/tasks/04_manage_user/app_password_drop.yml b/roles/web-app-stalwart/tasks/04_manage_user/app_password_drop.yml new file mode 100644 index 0000000000..9072532446 --- /dev/null +++ b/roles/web-app-stalwart/tasks/04_manage_user/app_password_drop.yml @@ -0,0 +1,15 @@ +--- +- name: "🗑️ Destroy the retired app passwords per bot account" + include_tasks: "{{ role_path }}/tasks/jmap_call.yml" + vars: + stalwart_jmap_act_as: "{{ stalwart_app_password_owner.key }}" + stalwart_jmap_body: >- + {{ {'using': ['urn:ietf:params:jmap:core', 'urn:stalwart:jmap'], + 'methodCalls': [['x:AppPassword/set', {'destroy': stalwart_app_password_owner.value}, 'c0']]} | to_json }} + loop: "{{ stalwart_app_password_stale | dict2items }}" + loop_control: + loop_var: stalwart_app_password_owner + +- name: "🧹 Clear the dropped app passwords" + ansible.builtin.set_fact: + stalwart_app_password_stale: {} diff --git a/roles/web-app-stalwart/tasks/06_oidc.yml b/roles/web-app-stalwart/tasks/06_oidc.yml index fda058a667..a042e1f990 100644 --- a/roles/web-app-stalwart/tasks/06_oidc.yml +++ b/roles/web-app-stalwart/tasks/06_oidc.yml @@ -86,28 +86,13 @@ stalwart_jmap_body: "{{ lookup('template', 'jmap/authentication/update.json.j2') }}" when: stalwart_auth_exists | bool -- name: "⚙️ Build the trusted-network relay expression" - ansible.builtin.set_fact: - stalwart_relay_expr: >- - !is_empty(authenticated_as){% for net in STALWART_TRUSTED_NETWORKS %} || remote_ip == '{{ net }}'{% endfor %} - - name: "📖 Read the RCPT stage singleton" include_tasks: jmap_call.yml vars: stalwart_jmap_body: "{{ lookup('template', 'jmap/rcpt/get.json.j2') }}" -- name: "🔧 Record whether the RCPT stage singleton exists" - ansible.builtin.set_fact: - stalwart_rcpt_exists: "{{ (stalwart_jmap.methodResponses[0][1].list | length) > 0 }}" - -- name: "📁 Create the RCPT stage with trusted-network relay" - include_tasks: jmap_call.yml - vars: - stalwart_jmap_body: "{{ lookup('template', 'jmap/rcpt/create.json.j2') }}" - when: not (stalwart_rcpt_exists | bool) - -- name: "🔄 Update the RCPT stage with trusted-network relay" +- name: "🔒 Restrict relaying to authenticated sessions" include_tasks: jmap_call.yml vars: stalwart_jmap_body: "{{ lookup('template', 'jmap/rcpt/update.json.j2') }}" - when: stalwart_rcpt_exists | bool + when: (stalwart_jmap.methodResponses[0][1].list | length) > 0 diff --git a/roles/web-app-stalwart/tasks/jmap_call.yml b/roles/web-app-stalwart/tasks/jmap_call.yml index d41404ff6d..2435c2e924 100644 --- a/roles/web-app-stalwart/tasks/jmap_call.yml +++ b/roles/web-app-stalwart/tasks/jmap_call.yml @@ -1,6 +1,7 @@ --- # Reusable management JMAP call: in `stalwart_jmap_body` (JSON), out `stalwart_jmap`. # Param: stalwart_jmap_tolerate_rejected - true when the caller reads a set-level rejection itself (e.g. alreadyExists). +# Param: stalwart_jmap_act_as - account address the recovery admin acts for; unset runs the call as the admin itself. - name: "🔧 Stalwart JMAP call" ansible.builtin.shell: cmd: >- @@ -9,7 +10,7 @@ -H "Content-Type:application/json" {{ STALWART_JMAP_URL }} stdin: |- - user = {{ (STALWART_ADMIN_USER ~ ':' ~ STALWART_ADMIN_PASSWORD) | to_json(ensure_ascii=False) }} + user = {{ ((stalwart_jmap_act_as ~ '%' if stalwart_jmap_act_as is defined else '') ~ STALWART_ADMIN_USER ~ ':' ~ STALWART_ADMIN_PASSWORD) | to_json(ensure_ascii=False) }} data-binary = {{ stalwart_jmap_body | to_json(ensure_ascii=False) }} register: stalwart_jmap_raw changed_when: false @@ -42,4 +43,6 @@ stalwart_jmap_rejected: >- {{ {} if (stalwart_jmap_tolerate_rejected | default(false) | bool) else (stalwart_jmap.methodResponses[0][1].notCreated | default({}, true)) - | combine(stalwart_jmap.methodResponses[0][1].notUpdated | default({}, true)) }} + | combine(stalwart_jmap.methodResponses[0][1].notUpdated | default({}, true)) + | combine(stalwart_jmap.methodResponses[0][1].notDestroyed | default({}, true) + | dict2items | rejectattr('value.type', 'equalto', 'notFound') | items2dict) }} diff --git a/roles/web-app-stalwart/tasks/main.yml b/roles/web-app-stalwart/tasks/main.yml index 83ba771fa2..a1b4eb324e 100644 --- a/roles/web-app-stalwart/tasks/main.yml +++ b/roles/web-app-stalwart/tasks/main.yml @@ -4,11 +4,20 @@ vars: application_id: "web-app-stalwart" -- include_tasks: 11_oidc_restart.yml +- name: "📥 Run the work deferred to the service-loader post-load flush" when: - run_once_web_app_stalwart is defined - - stalwart_oidc_restart_pending | default(false) | bool - not (service_loader_preloading | default(false) | bool) - IS_STACK_HOST | bool - vars: - application_id: "web-app-stalwart" + block: + - name: "🗑️ Drop the retired app passwords" + include_tasks: 04_manage_user/app_password_drop.yml + when: stalwart_app_password_stale | length > 0 + vars: + application_id: "web-app-stalwart" + + - name: "🔄 Restart Stalwart to activate the OIDC directory" + include_tasks: 11_oidc_restart.yml + when: stalwart_oidc_restart_pending | default(false) | bool + vars: + application_id: "web-app-stalwart" diff --git a/roles/web-app-stalwart/tasks/manager_ops.yml b/roles/web-app-stalwart/tasks/manager_ops.yml index 4727bd0c5d..a6d57d500a 100644 --- a/roles/web-app-stalwart/tasks/manager_ops.yml +++ b/roles/web-app-stalwart/tasks/manager_ops.yml @@ -27,6 +27,9 @@ - name: "🔧 Point Stalwart's file tracer at a writable directory" include_tasks: tracing.yml +- name: "⏱️ Let Stalwart's address bans expire" + include_tasks: security.yml + - name: "⚙️ Provision the Stalwart WebUI (WebAdmin) bundle" include_tasks: 02_webadmin.yml diff --git a/roles/web-app-stalwart/tasks/security.yml b/roles/web-app-stalwart/tasks/security.yml new file mode 100644 index 0000000000..a311f8a242 --- /dev/null +++ b/roles/web-app-stalwart/tasks/security.yml @@ -0,0 +1,34 @@ +--- +- name: "📖 Read Stalwart's ban settings" + include_tasks: jmap_call.yml + vars: + stalwart_jmap_body: >- + {{ {'using': ['urn:ietf:params:jmap:core', 'urn:stalwart:jmap'], + 'methodCalls': [['x:Security/get', {'ids': ['singleton']}, 'c0']]} | to_json }} + +- name: "🔍 Resolve Stalwart's current ban periods" + ansible.builtin.set_fact: + stalwart_ban_periods_current: >- + {{ stalwart_jmap.methodResponses[0][1].list[0] + | dict2items | selectattr('key', 'in', STALWART_BAN_PERIODS) | items2dict }} + +- name: "⏱️ Let Stalwart's address bans expire" + when: stalwart_ban_periods_current != STALWART_BAN_PERIODS + block: + - name: "🔧 Set Stalwart's ban periods" + include_tasks: jmap_call.yml + vars: + stalwart_jmap_body: >- + {{ {'using': ['urn:ietf:params:jmap:core', 'urn:stalwart:jmap'], + 'methodCalls': [['x:Security/set', + {'update': {'singleton': STALWART_BAN_PERIODS}}, + 'c0']]} | to_json }} + + - name: "🔄 Reload Stalwart's settings" + include_tasks: jmap_call.yml + vars: + stalwart_jmap_body: >- + {{ {'using': ['urn:ietf:params:jmap:core', 'urn:stalwart:jmap'], + 'methodCalls': [['x:Action/set', + {'create': {'a': {'@type': 'ReloadSettings'}}}, + 'c0']]} | to_json }} diff --git a/roles/web-app-stalwart/tasks/tracing.yml b/roles/web-app-stalwart/tasks/tracing.yml index 3057414b94..f968f3fa63 100644 --- a/roles/web-app-stalwart/tasks/tracing.yml +++ b/roles/web-app-stalwart/tasks/tracing.yml @@ -6,12 +6,18 @@ {{ {'using': ['urn:ietf:params:jmap:core', 'urn:stalwart:jmap'], 'methodCalls': [['x:Tracer/get', {'ids': none}, 'c0']]} | to_json }} -- name: "🔍 Resolve the file tracer" +- name: "🔍 Resolve the file tracer and the ban tracer" ansible.builtin.set_fact: stalwart_log_tracers: >- {{ stalwart_jmap.methodResponses[0][1].list | selectattr('@type', 'equalto', 'Log') | list }} + stalwart_ban_tracers: >- + {{ stalwart_jmap.methodResponses[0][1].list + | selectattr('@type', 'equalto', 'Stdout') + | selectattr('eventsPolicy', 'equalto', 'include') + | selectattr('events', 'superset', STALWART_BAN_TRACER.events | list) + | list }} - name: "🔧 Point the file tracer at a directory that exists" include_tasks: jmap_call.yml @@ -22,3 +28,11 @@ {'update': {stalwart_log_tracers[0].id: {'path': STALWART_LOG_DIR}}}, 'c0']]} | to_json }} when: stalwart_log_tracers | length > 0 + +- name: "📣 Print bans and failed logins on the container output" + include_tasks: jmap_call.yml + vars: + stalwart_jmap_body: >- + {{ {'using': ['urn:ietf:params:jmap:core', 'urn:stalwart:jmap'], + 'methodCalls': [['x:Tracer/set', {'create': {'t': STALWART_BAN_TRACER}}, 'c0']]} | to_json }} + when: stalwart_ban_tracers | length == 0 diff --git a/roles/web-app-stalwart/templates/jmap/rcpt/create.json.j2 b/roles/web-app-stalwart/templates/jmap/rcpt/create.json.j2 deleted file mode 100644 index a5df60533b..0000000000 --- a/roles/web-app-stalwart/templates/jmap/rcpt/create.json.j2 +++ /dev/null @@ -1,4 +0,0 @@ -{ - "using": ["urn:ietf:params:jmap:core", "urn:stalwart:jmap"], - "methodCalls": [["x:MtaStageRcpt/set", {"create": {"r": {"allowRelaying": {"else": {{ stalwart_relay_expr | trim | to_json }} }}}}, "c0"]] -} diff --git a/roles/web-app-stalwart/templates/jmap/rcpt/update.json.j2 b/roles/web-app-stalwart/templates/jmap/rcpt/update.json.j2 index 027616add2..b850276ba9 100644 --- a/roles/web-app-stalwart/templates/jmap/rcpt/update.json.j2 +++ b/roles/web-app-stalwart/templates/jmap/rcpt/update.json.j2 @@ -1,4 +1,4 @@ { "using": ["urn:ietf:params:jmap:core", "urn:stalwart:jmap"], - "methodCalls": [["x:MtaStageRcpt/set", {"update": {"singleton": {"allowRelaying": {"else": {{ stalwart_relay_expr | trim | to_json }} }}}}, "c0"]] + "methodCalls": [["x:MtaStageRcpt/set", {"update": {"singleton": {"allowRelaying": {"else": {{ STALWART_RELAY_RULE | to_json }} }}}}, "c0"]] } diff --git a/roles/web-app-stalwart/templates/test.env.j2 b/roles/web-app-stalwart/templates/test.env.j2 index e74416f36a..2076165c5f 100644 --- a/roles/web-app-stalwart/templates/test.env.j2 +++ b/roles/web-app-stalwart/templates/test.env.j2 @@ -12,8 +12,14 @@ MAILU_SMTP_CONTAINER={{ lookup('config', 'web-app-mailu', 'services.smtp.name') MAIL_DOMAIN={{ STALWART_DOMAIN | quote }} +MAIL_HOST={{ lookup('email').host | quote }} + MIGRATION_SCRIPT={{ lookup('path_absolute', 'roles', application_id, 'files/python/migrate_from_mailu.py') | quote }} +NO_REPLY_EMAIL={{ lookup('users', 'no-reply').email | quote }} + +NO_REPLY_SMTP_PASSWORD={{ lookup('email').password | quote }} + PYTHON_BIN={{ (lookup('env', 'PYTHON') or '/opt/venvs/infinito/bin/python') | quote }} STALWART_MIGRATION_E2E={{ ((application_id in PRIMARY_APPS) and ('web-app-mailu' in group_names) and DEPLOYMENT_MODE == 'compose' and IS_STACK_HOST | bool) | string | lower }} diff --git a/roles/web-app-stalwart/vars/main.yml b/roles/web-app-stalwart/vars/main.yml index 4d4cb42c1b..a3b02d46e1 100644 --- a/roles/web-app-stalwart/vars/main.yml +++ b/roles/web-app-stalwart/vars/main.yml @@ -12,6 +12,17 @@ container_port: "{{ lookup('config', application_id, 'services.stalwart.ports.in STALWART_LOCAL_BASE_URL: "http://localhost:{{ container_port }}" STALWART_JMAP_URL: "{{ STALWART_LOCAL_BASE_URL }}/jmap/" STALWART_LOG_DIR: "{{ lookup('volume', application_id, 'data').target }}" +STALWART_BAN_TRACER: + "@type": "Stdout" + buffered: false + level: "debug" + eventsPolicy: "include" + events: + "security.authentication-ban": true + "security.abuse-ban": true + "security.scan-ban": true + "security.loiter-ban": true + "auth.failed": true STALWART_JMAP_APPLICATION_GET: >- {{ {'using': ['urn:ietf:params:jmap:core', 'urn:stalwart:jmap'], 'methodCalls': [['x:Application/get', {'ids': none}, 'c0']]} | to_json }} @@ -65,7 +76,17 @@ STALWART_USERS: "{{ lookup('stalwart_users') }}" STALWART_DMARC_RUF: "{{ lookup('config', application_id, 'users.administrator.email') }}" -STALWART_TRUSTED_NETWORKS: "{{ NETWORK_TRUSTED_PROXY_CIDRS }}" +STALWART_BOT_APP_PASSWORD_DESCRIPTION: "Infinito.Nexus submission" +STALWART_BOT_APP_PASSWORD_RETIRED_DESCRIPTION: "Infinito.Nexus submission, retired" +stalwart_app_password_stale: {} +STALWART_RELAY_RULE: "!is_empty(authenticated_as)" + +STALWART_BAN_PERIOD_MS: "{{ (lookup('config', application_id, 'services.stalwart.ban_period_seconds') | int) * 1000 }}" +STALWART_BAN_PERIODS: + authBanPeriod: "{{ STALWART_BAN_PERIOD_MS | int }}" + abuseBanPeriod: "{{ STALWART_BAN_PERIOD_MS | int }}" + loiterBanPeriod: "{{ STALWART_BAN_PERIOD_MS | int }}" + scanBanPeriod: "{{ STALWART_BAN_PERIOD_MS | int }}" STALWART_SSO_ENABLED: "{{ lookup('sso', application_id, 'is_enabled') | bool }}" diff --git a/roles/web-app-taiga/meta/services.yml b/roles/web-app-taiga/meta/services.yml index 08c297fad0..3d7f2620b3 100644 --- a/roles/web-app-taiga/meta/services.yml +++ b/roles/web-app-taiga/meta/services.yml @@ -93,7 +93,7 @@ gateway: internal: http: 80 image: nginx - version: 1.31.6-alpine + version: 1.31.7-alpine cpus: "0.3" mem_reservation: 256m mem_limit: 512m diff --git a/roles/web-svc-coturn/meta/services.yml b/roles/web-svc-coturn/meta/services.yml index 26f1a9818f..6f10a65426 100644 --- a/roles/web-svc-coturn/meta/services.yml +++ b/roles/web-svc-coturn/meta/services.yml @@ -31,7 +31,7 @@ coturn: lifecycle: beta gateway: image: nginx - version: 1.31.6-alpine + version: 1.31.7-alpine cpus: "0.3" mem_reservation: 256m mem_limit: 512m diff --git a/roles/web-svc-logout/meta/services.yml b/roles/web-svc-logout/meta/services.yml index df98ea8867..bb9722bb27 100644 --- a/roles/web-svc-logout/meta/services.yml +++ b/roles/web-svc-logout/meta/services.yml @@ -12,7 +12,7 @@ logout: enabled: true shared: true image: ghcr.io/kevinveenbirkenbach/universal-logout - version: v1.4.1 + version: v1.4.2 name: logout min_storage: 250MB cpus: 0.2 diff --git a/roles/web-svc-onlyoffice/meta/services.yml b/roles/web-svc-onlyoffice/meta/services.yml index 7a0bbfbe86..367ffbcd6d 100644 --- a/roles/web-svc-onlyoffice/meta/services.yml +++ b/roles/web-svc-onlyoffice/meta/services.yml @@ -33,7 +33,7 @@ onlyoffice: enabled: true # nocheck: dynamic-flag shared: true image: onlyoffice/documentserver - version: 9.4.0 + version: 10.0.0 name: onlyoffice min_storage: 1468MB cpus: 2 diff --git a/scripts/lint/mermaid.sh b/scripts/lint/mermaid.sh index 5dd9c0fefd..1f1a606485 100755 --- a/scripts/lint/mermaid.sh +++ b/scripts/lint/mermaid.sh @@ -20,20 +20,6 @@ if ! command -v mmdc >/dev/null 2>&1; then exit 127 fi -# Limitation: install-lint skips the puppeteer browser download so it stays -# sandbox-safe; mermaid-cli (puppeteer-core) bundles none, so provision the -# matching chrome-headless-shell here. --install-deps also pulls chrome's shared -# libs (libnspr4, libnss3, ...) on root Linux. Idempotent; needs network + unzip. -if command -v apt-get >/dev/null 2>&1; then - apt-get -o DPkg::Lock::Timeout=600 update >/dev/null 2>&1 || - echo "Warning: apt-get update failed; chrome's shared libs may stay unresolved." >&2 -fi - -if ! deps_output="$(npx --yes puppeteer browsers install chrome-headless-shell --install-deps 2>&1)"; then - printf 'Warning: chrome-headless-shell provisioning failed; mermaid rendering may fail:\n%s\n' \ - "${deps_output}" >&2 -fi - workdir="$(mktemp -d)" trap 'rm -rf "${workdir}"' EXIT @@ -47,6 +33,15 @@ if [[ ${#md_files[@]} -eq 0 ]]; then exit 0 fi +printf '%s\n' 'flowchart LR' ' a --> b' >"${workdir}/preflight.mmd" +if ! mmdc --puppeteerConfigFile "${puppeteer_cfg}" -i "${workdir}/preflight.mmd" \ + -o "${workdir}/preflight.svg" >"${workdir}/preflight.log" 2>&1; then + printf "mmdc cannot render a minimal diagram, so none of the %d Markdown file(s) was checked. 'make install-lint-force' provisions its browser:\n" \ + "${#md_files[@]}" >&2 + sed 's/^/ /' "${workdir}/preflight.log" >&2 + exit 1 +fi + : "${INFINITO_WORKER_CPU:?INFINITO_WORKER_CPU must be set (provided by default.env via the env loader)}" jobs="${INFINITO_WORKER_CPU}" diff --git a/scripts/meta/env/load.sh b/scripts/meta/env/load.sh index b1dff51f4c..dfb67daf3a 100755 --- a/scripts/meta/env/load.sh +++ b/scripts/meta/env/load.sh @@ -7,6 +7,9 @@ # re-entry from generator subshells (BASH_ENV would otherwise recurse). # PYTHON/PATH are re-resolved on every source so a venv created mid-chain wins; # caller-set values are snapshotted and restored after the bulk .env source. +# INFINITO_CONTAINER, INFINITO_PARENT_IMAGE and INFINITO_IMAGE derive from the +# distro: a caller INFINITO_DISTRO other than the one `.env` was generated for +# loads a set generated for that distro instead, and `.env` stays untouched. # zsh cannot run this loader's bash-isms (BASH_SOURCE, ${!indirect}, assoc-array # key expansion). When sourced from zsh, delegate to bash and import the @@ -44,7 +47,7 @@ if [[ -n "${VENV:-}" && -d "${VENV%/}/bin" ]]; then fi _infinito_env_mtime="$(stat -c %Y "${_infinito_env_dotenv}" 2>/dev/null || echo 0)" -if [[ "${INFINITO_ENV_LOADED:-}" == "1" && "${INFINITO_ENV_LOADED_MTIME:-}" == "${_infinito_env_mtime}" ]]; then # nocheck: env-loader-internal-guard +if [[ "${INFINITO_ENV_LOADED:-}" == "1" && "${INFINITO_ENV_LOADED_MTIME:-}" == "${_infinito_env_mtime}" && "${INFINITO_ENV_LOADED_DISTRO:-}" == "${INFINITO_DISTRO:-}" ]]; then # nocheck: env-loader-internal-guard unset _infinito_env_repo_root _infinito_env_dotenv _infinito_env_mtime return 0 fi @@ -53,11 +56,23 @@ if [[ "${INFINITO_ENV_GENERATING:-}" == "1" ]]; then # nocheck: env-loader-inter return 0 fi -if [[ ! -f "${_infinito_env_dotenv}" ]]; then +if [[ -n "${INFINITO_ENV_LOADED_DISTRO:-}" && "${INFINITO_ENV_LOADED_DISTRO}" != "${INFINITO_DISTRO:-}" ]]; then # nocheck: env-loader-internal-guard + unset INFINITO_CONTAINER INFINITO_PARENT_IMAGE INFINITO_IMAGE +fi + +_infinito_env_source="${_infinito_env_dotenv}" +_infinito_env_scratch="" +_infinito_env_distro="${INFINITO_DISTRO:-}" +if [[ -f "${_infinito_env_dotenv}" && -n "${_infinito_env_distro}" && "${_infinito_env_distro}" != "$(sed -n 's/^INFINITO_DISTRO=//p' "${_infinito_env_dotenv}")" ]]; then + _infinito_env_scratch="$(mktemp -d)" + _infinito_env_source="${_infinito_env_scratch}/env" +fi + +if [[ ! -f "${_infinito_env_source}" ]]; then ( cd "${_infinito_env_repo_root}" || exit 1 export INFINITO_ENV_GENERATING=1 - "${PYTHON}" -m cli.meta.env + "${PYTHON}" -m cli.meta.env --output "${_infinito_env_source}" ) >&2 fi @@ -73,14 +88,19 @@ while IFS= read -r _infinito_env_line; do if [[ -n "${!_infinito_env_key:-}" ]]; then _infinito_env_preserved["${_infinito_env_key}"]="${!_infinito_env_key}" fi -done <"${_infinito_env_dotenv}" +done <"${_infinito_env_source}" unset _infinito_env_line _infinito_env_key set -a # shellcheck disable=SC1090 -source "${_infinito_env_dotenv}" +source "${_infinito_env_source}" set +a +if [[ -n "${_infinito_env_scratch}" ]]; then + rm -rf "${_infinito_env_scratch}" +fi +unset _infinito_env_source _infinito_env_scratch _infinito_env_distro + for _infinito_env_key in "${!_infinito_env_preserved[@]}"; do declare -gx "${_infinito_env_key}=${_infinito_env_preserved[${_infinito_env_key}]}" done @@ -100,4 +120,5 @@ fi export INFINITO_ENV_LOADED="1" _infinito_env_mtime="$(stat -c %Y "${_infinito_env_dotenv}" 2>/dev/null || echo 0)" export INFINITO_ENV_LOADED_MTIME="${_infinito_env_mtime}" +export INFINITO_ENV_LOADED_DISTRO="${INFINITO_DISTRO:-}" unset _infinito_env_repo_root _infinito_env_dotenv _infinito_env_mtime diff --git a/scripts/meta/resolve/missing/highest_release_tag.sh b/scripts/meta/resolve/missing/highest_release_tag.sh index 5fbefb79dc..70363b940d 100755 --- a/scripts/meta/resolve/missing/highest_release_tag.sh +++ b/scripts/meta/resolve/missing/highest_release_tag.sh @@ -1,6 +1,11 @@ #!/usr/bin/env bash set -euo pipefail +# The backfill stops below v9.0.2: v9.0.1 cannot build its image from its own +# tree (`infinito: command not found`, fixed by c680be12ea in v9.0.2), and +# v9.0.0 has the same Dockerfile, Makefile and scripts/docker. +oldest_buildable_tag="v9.0.2" + : "${OWNER:?Missing OWNER}" : "${REGISTRY:?Missing REGISTRY}" : "${REPO_PREFIX:?Missing REPO_PREFIX}" @@ -18,6 +23,9 @@ fi for ((i = ${#tags[@]} - 1; i >= 0; i--)); do tag="${tags[$i]}" + if [[ "$(printf '%s\n' "${tag}" "${oldest_buildable_tag}" | sort -V | head -n 1)" != "${oldest_buildable_tag}" ]]; then + exit 0 + fi tag_missing="false" for distro in ${INFINITO_DISTROS}; do diff --git a/scripts/tests/deploy/act/down_act_outer.sh b/scripts/tests/deploy/act/down_act_outer.sh index af792d57f2..f8d0340ae2 100755 --- a/scripts/tests/deploy/act/down_act_outer.sh +++ b/scripts/tests/deploy/act/down_act_outer.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash set +e -# Match act's outer-runner name prefix, not the image: ACT_PLATFORM_IMAGE varies the ancestor. -ids="$(docker ps -aq --filter "name=^act-")" +: "${INFINITO_CONTAINER:?INFINITO_CONTAINER is not set; source scripts/meta/env/load.sh or invoke via make}" + +ids="$(docker ps -aq --filter "name=^act-" --filter "label=infinito.worktree=${INFINITO_CONTAINER}")" if [ -n "${ids}" ]; then - echo "→ removing act outer container(s): ${ids}" + echo "→ removing act outer container(s) of ${INFINITO_CONTAINER}: ${ids}" # shellcheck disable=SC2086 docker rm -f ${ids} >/dev/null 2>&1 else diff --git a/scripts/tests/deploy/act/workflow.sh b/scripts/tests/deploy/act/workflow.sh index 2d5e126a9a..7be72c21a6 100755 --- a/scripts/tests/deploy/act/workflow.sh +++ b/scripts/tests/deploy/act/workflow.sh @@ -5,7 +5,8 @@ set -euo pipefail : "${ACT_EVENT:=workflow_dispatch}" : "${ACT_JOB:=}" : "${ACT_MATRIX:=}" -: "${ACT_CONTAINER_OPTIONS:=--privileged}" +: "${INFINITO_CONTAINER:?INFINITO_CONTAINER is not set; source scripts/meta/env/load.sh or invoke via make}" +: "${ACT_CONTAINER_OPTIONS:=--privileged --label infinito.worktree=${INFINITO_CONTAINER}}" : "${ACT_NETWORK:=host}" : "${ACT_PULL:=false}" : "${ACT_RM:=true}" diff --git a/tests/lint/ansible/roles/test_server_meta_no_split_fields.py b/tests/lint/ansible/roles/test_server_meta_no_split_fields.py index dc99f6f18f..3438671176 100644 --- a/tests/lint/ansible/roles/test_server_meta_no_split_fields.py +++ b/tests/lint/ansible/roles/test_server_meta_no_split_fields.py @@ -16,7 +16,7 @@ open. From 2026-12-28 it deliberately fails with a reminder to delete it. from __future__ import annotations -import datetime +import datetime as dt import unittest from utils.cache.yaml import load_yaml_any @@ -25,12 +25,12 @@ from utils.roles.mapping import ROLE_FILE_META_SERVER from . import PROJECT_ROOT _SPLIT_FIELDS: tuple[str, ...] = ("csp", "domains", "networks") -_SUNSET = datetime.date(2026, 12, 28) +_SUNSET = dt.date(2026, 12, 28) class TestServerMetaNoSplitFields(unittest.TestCase): def test_server_yml_has_no_split_fields(self) -> None: - if datetime.datetime.now(tz=datetime.UTC).date() >= _SUNSET: + if dt.datetime.now(tz=dt.UTC).date() >= _SUNSET: self.fail( "Migration guard expired (>= 2026-12-28): the meta/server.yml " "-> csp/domains/networks.yml split is long merged. DELETE this " diff --git a/tests/unit/python/cli/meta/ci/test_report_failures.py b/tests/unit/python/cli/meta/ci/test_report_failures.py index bfe7deed6c..c4c0bf4d22 100644 --- a/tests/unit/python/cli/meta/ci/test_report_failures.py +++ b/tests/unit/python/cli/meta/ci/test_report_failures.py @@ -3,6 +3,7 @@ from __future__ import annotations import tempfile import unittest from pathlib import Path +from unittest.mock import patch from cli.meta.ci.report_failures import ( Failure, @@ -10,6 +11,7 @@ from cli.meta.ci.report_failures import ( decisive_excerpt, failed_roles, issue_body, + list_jobs, ) @@ -92,6 +94,19 @@ class TestReportFailures(unittest.TestCase): artifact_name("web-app-x", Failure("compose", "0", "tor", "fedora", "zfs")), ) + def test_list_jobs_reads_one_job_per_line_across_pages(self) -> None: + pages = ( + '{"name": "deploy a", "conclusion": "success"}\n' + '{"name": "deploy b", "conclusion": "failure"}\n' + ) + with patch("cli.meta.ci.report_failures._gh", return_value=pages): + jobs = list_jobs("1", "owner/repo") + self.assertEqual([job["name"] for job in jobs], ["deploy a", "deploy b"]) + + def test_list_jobs_of_an_empty_listing_is_empty(self) -> None: + with patch("cli.meta.ci.report_failures._gh", return_value=""): + self.assertEqual(list_jobs("1", "owner/repo"), []) + def test_issue_body_lists_failures_and_run(self) -> None: body = issue_body( "web-app-xwiki", diff --git a/tests/unit/python/cli/meta/env/test_main.py b/tests/unit/python/cli/meta/env/test_main.py index d2cc44c42f..261857a54e 100644 --- a/tests/unit/python/cli/meta/env/test_main.py +++ b/tests/unit/python/cli/meta/env/test_main.py @@ -9,7 +9,13 @@ import cli.meta.env.__main__ as env_main class TestDotenvHostOwned(unittest.TestCase): - def _run(self, *, dotenv_exists: bool, host_owned: str | None) -> tuple[int, bool]: + def _run( + self, + *, + dotenv_exists: bool, + host_owned: str | None, + argv: list[str] | None = None, + ) -> tuple[int, bool]: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) (root / "default.env").write_text("INFINITO_DISTRO=debian\n") @@ -28,7 +34,7 @@ class TestDotenvHostOwned(unittest.TestCase): patch.object(env_main, "write_dotenv") as write_dotenv, ): build_env.return_value.values = {} - rc = env_main.main() + rc = env_main.main(argv) return rc, write_dotenv.called def test_a_host_owned_dotenv_is_kept(self) -> None: @@ -40,6 +46,16 @@ class TestDotenvHostOwned(unittest.TestCase): def test_without_the_marker_the_dotenv_is_regenerated(self) -> None: self.assertEqual(self._run(dotenv_exists=True, host_owned=None), (0, True)) + def test_an_explicit_output_is_written_even_when_host_owned(self) -> None: + self.assertEqual( + self._run( + dotenv_exists=True, + host_owned="true", + argv=["--output", "/nonexistent/distro.env"], + ), + (0, True), + ) + if __name__ == "__main__": # pragma: no cover unittest.main() diff --git a/tests/unit/python/plugins/lookup/test_current_play_redirect_domains.py b/tests/unit/python/plugins/lookup/test_current_play_redirect_domains.py index d10c59aef4..dfed38a0f2 100644 --- a/tests/unit/python/plugins/lookup/test_current_play_redirect_domains.py +++ b/tests/unit/python/plugins/lookup/test_current_play_redirect_domains.py @@ -351,6 +351,21 @@ class CurrentPlayRedirectDomainsLookupTests(unittest.TestCase): [{"source": DOMAIN, "target": "abcdefonion.onion"}], ) + def test_onion_apex_absent_when_homepage_is_clearnet(self): + lm = self._make_lookup() + variables = self._onion_variables( + DOMAIN_HOMEPAGE="infinito.nexus", + group_names=["svc-net-tor"], + ) + with self._patch_lookups( + deployed=["web-opt-rdr-domains"], current_play_apps={} + ): + result = lm.run(terms=[], variables=variables)[0] + self.assertEqual( + result, + [{"source": "infinito.test", "target": "infinito.nexus"}], + ) + def test_onion_apex_node_value_templated(self): class _ResolvingTemplar: def __init__(self, mapping): diff --git a/tests/unit/python/plugins/lookup/test_email.py b/tests/unit/python/plugins/lookup/test_email.py index 9e076edf68..2790227bd1 100644 --- a/tests/unit/python/plugins/lookup/test_email.py +++ b/tests/unit/python/plugins/lookup/test_email.py @@ -253,7 +253,7 @@ class TestEmailLookup(unittest.TestCase): _write_role_config( self._tmp, "web-app-mailprov", - {"sso": {"oidc": {"submission_via_relay": True}}}, + {"sso": {"enabled": True}}, ) variables = { "MAIL_PROVIDER": "web-app-mailprov", @@ -269,56 +269,11 @@ class TestEmailLookup(unittest.TestCase): self.assertFalse(result["start_tls"]) self.assertFalse(result["tls"]) - def test_sso_relay_provider_disables_auth_and_uses_port_25(self) -> None: - # submission_via_relay + Keycloak deployed -> relay on 25, no auth, STARTTLS. + def test_a_provider_behind_sso_keeps_authenticated_submission(self) -> None: _write_role_config( self._tmp, "web-app-mailprov", - {"sso": {"oidc": {"submission_via_relay": True}}}, - ) - variables = { - "MAIL_PROVIDER": "web-app-mailprov", - "group_names": ["web-app-mailprov", "web-app-keycloak"], - "groups": {"web-app-mailprov": ["host1"], "web-app-keycloak": ["host1"]}, - "TLS_ENABLED": True, - "inventory_hostname": "host1", - } - result = self.lookup.run( - [], variables=variables, roles_dir=str(self._tmp / "roles") - )[0] - self.assertEqual(result["port"], 25) - self.assertFalse(result["auth"]) - self.assertTrue(result["start_tls"]) - - def test_sso_relay_inactive_without_keycloak(self) -> None: - # Same provider, but Keycloak is not deployed: keep authenticated 465. - _write_role_config( - self._tmp, - "web-app-mailprov", - {"sso": {"oidc": {"submission_via_relay": True}}}, - ) - variables = { - "MAIL_PROVIDER": "web-app-mailprov", - "group_names": ["web-app-mailprov"], - "groups": {"web-app-mailprov": ["host1"]}, - "TLS_ENABLED": True, - "inventory_hostname": "host1", - } - result = self.lookup.run( - [], variables=variables, roles_dir=str(self._tmp / "roles") - )[0] - self.assertEqual(result["port"], 465) - self.assertTrue(result["auth"]) - self.assertFalse(result["start_tls"]) - - def test_sso_relay_inactive_when_sso_enabled_pinned_false(self) -> None: - # A variant pins the provider's sso.enabled to a literal false while - # submission_via_relay stays true (role default) and Keycloak is still - # deployed: the provider keeps password auth, so no relay mode. - _write_role_config( - self._tmp, - "web-app-mailprov", - {"sso": {"enabled": False, "oidc": {"submission_via_relay": True}}}, + {"sso": {"enabled": True}}, ) variables = { "MAIL_PROVIDER": "web-app-mailprov", @@ -332,6 +287,7 @@ class TestEmailLookup(unittest.TestCase): )[0] self.assertEqual(result["port"], 465) self.assertTrue(result["auth"]) + self.assertEqual(result["auth_mechanism"], "on") self.assertFalse(result["start_tls"]) def test_a_provider_without_plaintext_submission_falls_back_to_its_mx( @@ -396,33 +352,6 @@ class TestEmailLookup(unittest.TestCase): self.assertTrue(result["auth"]) self.assertEqual(result["auth_mechanism"], "plain") - def test_sso_relay_active_with_untemplated_enabled_gate(self) -> None: - # The role default gates sso.enabled on group_names (raw Jinja here); - # the guard must not treat the untemplated string as false. - _write_role_config( - self._tmp, - "web-app-mailprov", - { - "sso": { - "enabled": "{{ 'web-app-keycloak' in group_names }}", - "oidc": {"submission_via_relay": True}, - } - }, - ) - variables = { - "MAIL_PROVIDER": "web-app-mailprov", - "group_names": ["web-app-mailprov", "web-app-keycloak"], - "groups": {"web-app-mailprov": ["host1"], "web-app-keycloak": ["host1"]}, - "TLS_ENABLED": True, - "inventory_hostname": "host1", - } - result = self.lookup.run( - [], variables=variables, roles_dir=str(self._tmp / "roles") - )[0] - self.assertEqual(result["port"], 25) - self.assertFalse(result["auth"]) - self.assertTrue(result["start_tls"]) - def test_computed_defaults_are_templated(self) -> None: self.lookup._templar = _DummyTemplar( {"DOMAIN_PRIMARY_RESOLVED": "mail.example.org"} diff --git a/tests/unit/python/roles/svc-opt-keyboard-color/files/test_script.py b/tests/unit/python/roles/svc-opt-keyboard-color/files/test_script.py index 695b4af265..d61a6f9ff2 100755 --- a/tests/unit/python/roles/svc-opt-keyboard-color/files/test_script.py +++ b/tests/unit/python/roles/svc-opt-keyboard-color/files/test_script.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import datetime +import datetime as dt import importlib.util from unittest import TestCase, main @@ -70,19 +70,19 @@ class CalculateColorTests(TestCase): class GetCurrentPeriodTests(TestCase): def test_a_time_inside_a_period_returns_that_period_colors(self): self.assertEqual( - SCRIPT.get_current_period(datetime.time(12, 0), COLOR_TIMES), + SCRIPT.get_current_period(dt.time(12, 0), COLOR_TIMES), ("ff0000", "00ff00"), ) def test_a_time_before_the_first_period_wraps_to_the_last(self): self.assertEqual( - SCRIPT.get_current_period(datetime.time(3, 0), COLOR_TIMES), + SCRIPT.get_current_period(dt.time(3, 0), COLOR_TIMES), ("0000ff", "ff0000"), ) def test_a_time_after_the_last_period_returns_the_first(self): self.assertEqual( - SCRIPT.get_current_period(datetime.time(23, 0), COLOR_TIMES), + SCRIPT.get_current_period(dt.time(23, 0), COLOR_TIMES), ("ff0000", "00ff00"), ) @@ -91,7 +91,7 @@ class CalculateTransitionRatioTests(TestCase): def test_the_start_of_a_window_is_ratio_zero(self): self.assertEqual( SCRIPT.calculate_transition_ratio( - datetime.time(6, 0), datetime.time(6, 0), datetime.time(12, 0) + dt.time(6, 0), dt.time(6, 0), dt.time(12, 0) ), 0, ) @@ -99,7 +99,7 @@ class CalculateTransitionRatioTests(TestCase): def test_the_end_of_a_window_is_ratio_one(self): self.assertEqual( SCRIPT.calculate_transition_ratio( - datetime.time(12, 0), datetime.time(6, 0), datetime.time(12, 0) + dt.time(12, 0), dt.time(6, 0), dt.time(12, 0) ), 1, ) @@ -107,7 +107,7 @@ class CalculateTransitionRatioTests(TestCase): def test_the_middle_of_a_window_is_ratio_one_half(self): self.assertEqual( SCRIPT.calculate_transition_ratio( - datetime.time(9, 0), datetime.time(6, 0), datetime.time(12, 0) + dt.time(9, 0), dt.time(6, 0), dt.time(12, 0) ), 0.5, ) @@ -115,7 +115,7 @@ class CalculateTransitionRatioTests(TestCase): def test_a_zero_length_window_returns_zero_instead_of_dividing_by_it(self): self.assertEqual( SCRIPT.calculate_transition_ratio( - datetime.time(6, 0), datetime.time(6, 0), datetime.time(6, 0) + dt.time(6, 0), dt.time(6, 0), dt.time(6, 0) ), 0, ) diff --git a/tests/unit/python/roles/sys-svc-container/files/test_apt_repo.py b/tests/unit/python/roles/sys-svc-container/files/test_apt_repo.py new file mode 100644 index 0000000000..66abd7521e --- /dev/null +++ b/tests/unit/python/roles/sys-svc-container/files/test_apt_repo.py @@ -0,0 +1,100 @@ +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils import PROJECT_ROOT +from utils.cache.files import read_text + +SCRIPT = PROJECT_ROOT / "roles/sys-svc-container/files/shell/apt_repo.sh" + +CURL_STUB = """#!/bin/sh +dest="" +while [ "$#" -gt 0 ]; do +\tif [ "$1" = "-o" ]; then +\t\tdest="$2" +\t\tshift +\tfi +\tshift +done +if [ -z "$STUB_SERVED_KEY" ]; then +\techo "curl: (7) Failed to connect to download.docker.com port 443 after 0 ms" >&2 +\texit 7 +fi +cp "$STUB_SERVED_KEY" "$dest" +""" + +SUDO_STUB = '#!/bin/sh\nexec "$@"\n' + + +class TestAptRepoInstallKey(unittest.TestCase): + def run_install_key(self, *, installed, served): + """Run `apt_repo.sh install-key` against a stubbed key download. + + Args: + installed: content of the keyring before the run, or None when no + key is installed. + served: content the download returns, or None when the download + host refuses the connection. + + Returns: + Tuple of exit code, stdout, stderr and the keyring content after + the run (None when no keyring exists). + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bin_dir = root / "bin" + bin_dir.mkdir() + for name, body in (("curl", CURL_STUB), ("sudo", SUDO_STUB)): + stub = bin_dir / name + stub.write_text(body) + stub.chmod(0o755) + keyring_dir = root / "keyrings" + keyring_dir.mkdir() + keyring_dir.chmod(0o755) + key = keyring_dir / "docker.asc" + if installed is not None: + key.write_text(installed) + key.chmod(0o644) + env = dict(os.environ) + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + env["STUB_SERVED_KEY"] = "" + if served is not None: + served_file = root / "served.asc" + served_file.write_text(served) + env["STUB_SERVED_KEY"] = str(served_file) + done = subprocess.run( + ["bash", str(SCRIPT), "install-key", "debian", str(key)], + env=env, + capture_output=True, + text=True, + check=False, + ) + after = read_text(str(key)) if key.exists() else None + return done.returncode, done.stdout, done.stderr, after + + def test_a_refused_download_keeps_the_installed_key(self): + code, out, err, after = self.run_install_key(installed="OLD KEY\n", served=None) + self.assertEqual(code, 0, err) + self.assertIn("changed=0", out) + self.assertIn("keeping", err) + self.assertEqual(after, "OLD KEY\n") + + def test_a_refused_download_without_a_key_fails(self): + code, _, err, after = self.run_install_key(installed=None, served=None) + self.assertEqual(code, 1) + self.assertIn("Failed to download the Docker GPG key", err) + self.assertIsNone(after) + + def test_a_new_key_replaces_the_installed_one(self): + code, out, err, after = self.run_install_key( + installed="OLD KEY\n", served="NEW KEY\n" + ) + self.assertEqual(code, 0, err) + self.assertIn("changed=1", out) + self.assertEqual(after, "NEW KEY\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/roles/web-app-gitlab/test_smtp_settings.py b/tests/unit/python/roles/web-app-gitlab/test_smtp_settings.py index 52e8dd229a..43601a9a52 100644 --- a/tests/unit/python/roles/web-app-gitlab/test_smtp_settings.py +++ b/tests/unit/python/roles/web-app-gitlab/test_smtp_settings.py @@ -2,7 +2,7 @@ Ruby's ``net/smtp`` authenticates whenever a ``user_name`` or ``password`` is present, falling back to its default auth type — ``authentication: nil`` does -NOT switch it off. So on the provider's SSO relay, where port 25 offers no +NOT switch it off. So on the provider's MX listener, where port 25 offers no AUTH at all, leaving the credentials in the hash makes every send die with ``503 5.5.1 AUTH not allowed`` while GitLab still reports the mail as queued. The credentials therefore have to be omitted, not merely unaccompanied by an diff --git a/tests/unit/python/roles/web-app-n8n/files/provision/test_client.py b/tests/unit/python/roles/web-app-n8n/files/provision/test_client.py index d698b892dc..914ab8b83f 100644 --- a/tests/unit/python/roles/web-app-n8n/files/provision/test_client.py +++ b/tests/unit/python/roles/web-app-n8n/files/provision/test_client.py @@ -8,10 +8,26 @@ import io import json import unittest from unittest.mock import patch +from urllib.parse import parse_qs, urlsplit from . import PROJECT_ROOT SCRIPT_PATH = PROJECT_ROOT / "roles/web-app-n8n/files/python/provision/client.py" +PAGE_SIZE = 10 +WORKFLOW_PAGE_SIZE = 100 +MAX_PAGE_SIZE = 250 + + +def paged(items, offset, limit): + """Return one page the way n8n slices a listing. + + Args: + items: every record the listing holds. + offset: index of the first record on the page. + limit: requested page size, capped like n8n caps it. + """ + return items[offset : offset + min(limit, MAX_PAGE_SIZE)] + PROVIDER = { "id": "web-app-gitea", @@ -78,16 +94,34 @@ class FakeApi: self.deleted_credentials.append(dead) self.credentials = [c for c in self.credentials if c["id"] != dead] return 200, {"success": True} - if path == "/rest/api-keys" and method == "GET": - return 200, {"data": [dict(key) for key in self.keys]} + if path.startswith("/rest/api-keys?") and method == "GET": + query = parse_qs(urlsplit(path).query) + label = query["label"][0].lower() + items = [dict(key) for key in self.keys if label in key["label"].lower()] + skip = int(query.get("skip", ["0"])[0]) + take = int(query.get("take", [str(PAGE_SIZE)])[0]) + return 200, {"data": {"items": paged(items, skip, take)}} if path == "/rest/api-keys" and method == "POST": self.keys = [{"id": "k1", "label": payload["label"]}] return 200, {"data": {"id": "k1", "rawApiKey": "n8n-raw-key"}} if path.startswith("/rest/api-keys/") and method == "DELETE": self.deleted_keys.append(path.rsplit("/", 1)[-1]) return 200, {"success": True} - if path == "/api/v1/workflows" and method == "GET": - return 200, {"data": [dict(flow) for flow in self.workflows]} + if path.split("?")[0] == "/api/v1/workflows" and method == "GET": + query = parse_qs(urlsplit(path).query) + name = query.get("name", [""])[0] + hits = [dict(flow) for flow in self.workflows if name in flow["name"]] + cursor = query.get("cursor", [f"0:{WORKFLOW_PAGE_SIZE}"])[0].split(":") + offset, limit = (int(cursor[0]), int(cursor[1])) + if "limit" in query: + limit = int(query["limit"][0]) + page = paged(hits, offset, limit) + following = offset + len(page) + more = following < len(hits) + return 200, { + "data": page, + "nextCursor": f"{following}:{limit}" if more else None, + } if path == "/api/v1/workflows" and method == "POST": self.created_workflows.append(payload) return 200, dict(payload, id="w1", active=False) @@ -179,6 +213,100 @@ class TestProvisionClient(unittest.TestCase): module.main() self.assertEqual(["k1"], api.deleted_keys) + def test_a_stale_managed_key_behind_a_full_page_is_still_replaced(self) -> None: + module = load_script() + humans = [{"id": f"h{n}", "label": f"human {n}"} for n in range(PAGE_SIZE)] + api = FakeApi(keys=[*humans, {"id": "old", "label": "infinito:mcp"}]) + with patch.object(module, "call", api): + module.main() + self.assertEqual(["old", "k1"], api.deleted_keys) + + def test_a_stale_managed_key_behind_a_full_filtered_page_is_still_replaced( + self, + ) -> None: + module = load_script() + near = [ + {"id": f"n{n}", "label": f"infinito:mcp-copy-{n}"} + for n in range(MAX_PAGE_SIZE) + ] + api = FakeApi(keys=[*near, {"id": "old", "label": "infinito:mcp"}]) + with patch.object(module, "call", api): + module.main() + self.assertEqual(["old", "k1"], api.deleted_keys) + + def test_a_managed_workflow_on_a_later_page_is_updated_not_duplicated( + self, + ) -> None: + module = load_script() + near = [ + {"id": f"n{n}", "name": f"infinito:mcp-agent copy {n}"} + for n in range(MAX_PAGE_SIZE) + ] + managed = {"id": "w9", "name": "infinito:mcp-agent"} + api = FakeApi(workflows=[*near, managed], credentials=[dict(REFERENCE)]) + with patch.object(module, "call", api): + module.main() + self.assertEqual([], api.created_workflows) + self.assertEqual(1, len(api.updated_workflows)) + + def test_a_repeated_workflow_cursor_aborts_rather_than_loops(self) -> None: + module = load_script() + api = FakeApi() + listings = [] + + def stuck(path, method="GET", payload=None, api_key=None): + if path.startswith("/api/v1/workflows?"): + listings.append(path) + if len(listings) > 3: + raise AssertionError("the workflow listing never ends") + return 200, {"data": [], "nextCursor": "same"} + return api(path, method, payload, api_key) + + with ( + patch.object(module, "call", stuck), + self.assertRaises(SystemExit) as exit_, + ): + module.main() + self.assertIn("nextCursor repeats", str(exit_.exception)) + + def test_a_key_listing_that_ignores_skip_aborts_rather_than_loops(self) -> None: + module = load_script() + api = FakeApi() + full = [ + {"id": f"n{n}", "label": f"infinito:mcp-copy-{n}"} + for n in range(MAX_PAGE_SIZE) + ] + listings = [] + + def frozen(path, method="GET", payload=None, api_key=None): + if path.startswith("/rest/api-keys?"): + listings.append(path) + if len(listings) > 3: + raise AssertionError("the key listing never ends") + return 200, {"data": {"items": full}} + return api(path, method, payload, api_key) + + with ( + patch.object(module, "call", frozen), + self.assertRaises(SystemExit) as exit_, + ): + module.main() + self.assertIn("repeats", str(exit_.exception)) + + def test_an_unknown_listing_shape_names_its_keys_but_never_a_value(self) -> None: + module = load_script() + api = FakeApi() + + def odd(path, method="GET", payload=None, api_key=None): + if path == "/rest/credentials" and method == "GET": + return 200, {"data": {"token": "t0ps3cret"}} + return api(path, method, payload, api_key) + + with patch.object(module, "call", odd), self.assertRaises(SystemExit) as exit_: + module.main() + self.assertIn("['token']", str(exit_.exception)) + self.assertNotIn("t0ps3cret", str(exit_.exception)) + def test_a_rotated_provider_token_replaces_the_stored_one(self) -> None: module = load_script() api = FakeApi(credentials=[dict(REFERENCE)]) diff --git a/tests/unit/python/roles/web-app-n8n/files/provision/test_mcp.py b/tests/unit/python/roles/web-app-n8n/files/provision/test_mcp.py index 81dcecf03a..a664866a33 100644 --- a/tests/unit/python/roles/web-app-n8n/files/provision/test_mcp.py +++ b/tests/unit/python/roles/web-app-n8n/files/provision/test_mcp.py @@ -8,10 +8,26 @@ import io import json import unittest from unittest.mock import patch +from urllib.parse import parse_qs, urlsplit from . import PROJECT_ROOT SCRIPT_PATH = PROJECT_ROOT / "roles/web-app-n8n/files/python/provision/mcp.py" +PAGE_SIZE = 10 +WORKFLOW_PAGE_SIZE = 100 +MAX_PAGE_SIZE = 250 + + +def paged(items, offset, limit): + """Return one page the way n8n slices a listing. + + Args: + items: every record the listing holds. + offset: index of the first record on the page. + limit: requested page size, capped like n8n caps it. + """ + return items[offset : offset + min(limit, MAX_PAGE_SIZE)] + ENV = { "N8N_BASE": "http://n8n:5678", @@ -51,6 +67,7 @@ class FakeApi: self.deleted_keys = [] self.logins = 0 self.minted = 0 + self.scopes = set() def __call__(self, path, method="GET", payload=None, api_key=None): if path == "/rest/login": @@ -72,16 +89,35 @@ class FakeApi: self.deleted_credentials.append(dead) self.credentials = [c for c in self.credentials if c["id"] != dead] return 200, {"success": True} - if path == "/rest/api-keys" and method == "GET": - return 200, {"data": [dict(key) for key in self.keys]} + if path.startswith("/rest/api-keys?") and method == "GET": + query = parse_qs(urlsplit(path).query) + label = query["label"][0].lower() + items = [dict(key) for key in self.keys if label in key["label"].lower()] + skip = int(query.get("skip", ["0"])[0]) + take = int(query.get("take", [str(PAGE_SIZE)])[0]) + return 200, {"data": {"items": paged(items, skip, take)}} if path == "/rest/api-keys" and method == "POST": self.keys = [{"id": "k1", "label": payload["label"]}] + self.scopes = set(payload["scopes"]) return 200, {"data": {"id": "k1", "rawApiKey": "n8n-raw-key"}} if path.startswith("/rest/api-keys/") and method == "DELETE": self.deleted_keys.append(path.rsplit("/", 1)[-1]) return 200, {"success": True} - if path == "/api/v1/workflows" and method == "GET": - return 200, {"data": [dict(flow) for flow in self.workflows]} + if path.split("?")[0] == "/api/v1/workflows" and method == "GET": + query = parse_qs(urlsplit(path).query) + name = query.get("name", [""])[0] + hits = [dict(flow) for flow in self.workflows if name in flow["name"]] + cursor = query.get("cursor", [f"0:{WORKFLOW_PAGE_SIZE}"])[0].split(":") + offset, limit = (int(cursor[0]), int(cursor[1])) + if "limit" in query: + limit = int(query["limit"][0]) + page = paged(hits, offset, limit) + following = offset + len(page) + more = following < len(hits) + return 200, { + "data": page, + "nextCursor": f"{following}:{limit}" if more else None, + } if path == "/api/v1/workflows" and method == "POST": self.created_workflows.append(payload) return 200, dict(payload, id="w1", active=False) @@ -89,6 +125,8 @@ class FakeApi: self.updated_workflows.append(payload) return 200, payload if path.endswith("/activate") and method == "POST": + if "workflow:activate" not in self.scopes: + return 403, '{"message":"Forbidden"}' self.activated_workflows.append(path.split("/")[-2]) return 200, {"active": True} if path.startswith("/api/v1/workflows/") and method == "DELETE": @@ -184,6 +222,120 @@ class TestProvisionMcp(unittest.TestCase): module.main() self.assertEqual(["old", "k1"], api.deleted_keys) + def test_a_stale_managed_key_behind_a_full_page_is_still_replaced(self) -> None: + module = load_script() + humans = [{"id": f"h{n}", "label": f"human {n}"} for n in range(PAGE_SIZE)] + api = FakeApi(keys=[*humans, {"id": "old", "label": "infinito:mcp"}]) + with patch.object(module, "call", api): + module.main() + self.assertEqual(["old", "k1"], api.deleted_keys) + + def test_an_unknown_listing_shape_names_its_keys_but_never_a_value(self) -> None: + module = load_script() + api = FakeApi() + + def odd(path, method="GET", payload=None, api_key=None): + if path.startswith("/rest/api-keys?"): + return 200, {"data": {"secret": "t0ps3cret"}} + return api(path, method, payload, api_key) + + with patch.object(module, "call", odd), self.assertRaises(SystemExit) as exit_: + module.main() + self.assertIn("['secret']", str(exit_.exception)) + self.assertNotIn("t0ps3cret", str(exit_.exception)) + + def test_an_unpaged_listing_of_n8n_1_still_finds_the_stale_key(self) -> None: + module = load_script() + api = FakeApi() + + def unpaged(path, method="GET", payload=None, api_key=None): + if path.startswith("/rest/api-keys?"): + return 200, {"data": [{"id": "old", "label": "infinito:mcp"}]} + return api(path, method, payload, api_key) + + with patch.object(module, "call", unpaged): + module.main() + self.assertEqual(["old", "k1"], api.deleted_keys) + + def test_a_stale_managed_key_behind_a_full_filtered_page_is_still_replaced( + self, + ) -> None: + module = load_script() + near = [ + {"id": f"n{n}", "label": f"infinito:mcp-copy-{n}"} + for n in range(MAX_PAGE_SIZE) + ] + api = FakeApi(keys=[*near, {"id": "old", "label": "infinito:mcp"}]) + with patch.object(module, "call", api): + module.main() + self.assertEqual(["old", "k1"], api.deleted_keys) + + def test_a_managed_workflow_on_a_later_page_is_updated_not_duplicated( + self, + ) -> None: + module = load_script() + near = [ + {"id": f"n{n}", "name": f"infinito:mcp-server copy {n}"} + for n in range(MAX_PAGE_SIZE) + ] + managed = {"id": "w9", "name": "infinito:mcp-server", "active": True} + api = FakeApi(workflows=[*near, managed]) + with patch.object(module, "call", api): + module.main() + self.assertEqual([], api.created_workflows) + self.assertEqual(1, len(api.updated_workflows)) + + def test_a_repeated_workflow_cursor_aborts_rather_than_loops(self) -> None: + module = load_script() + api = FakeApi() + listings = [] + + def stuck(path, method="GET", payload=None, api_key=None): + if path.startswith("/api/v1/workflows?"): + listings.append(path) + if len(listings) > 3: + raise AssertionError("the workflow listing never ends") + return 200, {"data": [], "nextCursor": "same"} + return api(path, method, payload, api_key) + + with ( + patch.object(module, "call", stuck), + self.assertRaises(SystemExit) as exit_, + ): + module.main() + self.assertIn("nextCursor repeats", str(exit_.exception)) + + def test_a_key_listing_that_ignores_skip_aborts_rather_than_loops(self) -> None: + module = load_script() + api = FakeApi() + full = [ + {"id": f"n{n}", "label": f"infinito:mcp-copy-{n}"} + for n in range(MAX_PAGE_SIZE) + ] + listings = [] + + def frozen(path, method="GET", payload=None, api_key=None): + if path.startswith("/rest/api-keys?"): + listings.append(path) + if len(listings) > 3: + raise AssertionError("the key listing never ends") + return 200, {"data": {"items": full}} + return api(path, method, payload, api_key) + + with ( + patch.object(module, "call", frozen), + self.assertRaises(SystemExit) as exit_, + ): + module.main() + self.assertIn("repeats", str(exit_.exception)) + + def test_a_key_that_only_contains_the_managed_name_is_left_alone(self) -> None: + module = load_script() + api = FakeApi(keys=[{"id": "near", "label": "infinito:mcp-legacy"}]) + with patch.object(module, "call", api): + module.main() + self.assertEqual(["k1"], api.deleted_keys) + def test_the_run_leaves_no_api_key_behind(self) -> None: module = load_script() api = FakeApi() diff --git a/tests/unit/python/roles/web-app-prometheus/test_blackbox_ready_module.py b/tests/unit/python/roles/web-app-prometheus/test_blackbox_ready_module.py index c49856cbc6..230d024290 100644 --- a/tests/unit/python/roles/web-app-prometheus/test_blackbox_ready_module.py +++ b/tests/unit/python/roles/web-app-prometheus/test_blackbox_ready_module.py @@ -10,7 +10,9 @@ from utils.cache.yaml import load_yaml_str TEMPLATE_DIR = PROJECT_ROOT / "roles/web-app-prometheus/templates/configuration" -def ready_module(*, tls_mode: str, prometheus_tls: bool, ca_injected: bool) -> dict: +def ready_module( + *, name: str, tls_mode: str, prometheus_tls: bool, ca_injected: bool +) -> dict: def lookup(name, *terms): if name == "tls" and terms[1] == "enabled": return prometheus_tls @@ -29,33 +31,55 @@ def ready_module(*, tls_mode: str, prometheus_tls: bool, ca_injected: bool) -> d application_id="web-app-prometheus", TLS_MODE=tls_mode, ) - return load_yaml_str(rendered)["modules"]["http_ready"]["http"] + return load_yaml_str(rendered)["modules"][name]["http"] class TestBlackboxReadyModule(unittest.TestCase): def test_an_http_target_passes_while_prometheus_itself_uses_tls(self): module = ready_module( - tls_mode="letsencrypt", prometheus_tls=True, ca_injected=False + name="http_ready", + tls_mode="letsencrypt", + prometheus_tls=True, + ca_injected=False, ) self.assertIs(module["fail_if_not_ssl"], False) def test_certificates_are_verified_while_prometheus_itself_has_no_tls(self): module = ready_module( - tls_mode="letsencrypt", prometheus_tls=False, ca_injected=False + name="http_ready", + tls_mode="letsencrypt", + prometheus_tls=False, + ca_injected=False, ) self.assertIs(module["tls_config"]["insecure_skip_verify"], False) def test_certificates_are_verified_once_the_private_ca_is_mounted(self): module = ready_module( - tls_mode="self_signed", prometheus_tls=True, ca_injected=True + name="http_ready", + tls_mode="self_signed", + prometheus_tls=True, + ca_injected=True, ) self.assertIs(module["tls_config"]["insecure_skip_verify"], False) - def test_verification_is_skipped_only_without_the_private_ca(self): + def test_certificates_are_verified_even_without_the_private_ca(self): module = ready_module( - tls_mode="self_signed", prometheus_tls=False, ca_injected=False + name="http_ready", + tls_mode="self_signed", + prometheus_tls=False, + ca_injected=False, + ) + self.assertIs(module["tls_config"]["insecure_skip_verify"], False) + + def test_only_the_private_ca_module_skips_verification(self): + module = ready_module( + name="http_ready_private_ca", + tls_mode="self_signed", + prometheus_tls=False, + ca_injected=False, ) self.assertIs(module["tls_config"]["insecure_skip_verify"], True) + self.assertIs(module["fail_if_not_ssl"], False) if __name__ == "__main__": diff --git a/tests/unit/python/roles/web-app-prometheus/test_blackbox_targets.py b/tests/unit/python/roles/web-app-prometheus/test_blackbox_targets.py index ae4691a095..1f437ba169 100644 --- a/tests/unit/python/roles/web-app-prometheus/test_blackbox_targets.py +++ b/tests/unit/python/roles/web-app-prometheus/test_blackbox_targets.py @@ -16,35 +16,48 @@ DOMAINS = { "web-app-prometheus": [f"prometheus.{ONION}"], "web-app-nextcloud": {"nextcloud": f"next.cloud.{ONION}"}, "web-app-gitea": [f"git.{DOMAIN}"], + "web-app-matrix": [f"matrix.{DOMAIN}"], "web-svc-cdn": [f"cdn.{DOMAIN}"], "svc-db-postgres": [f"postgres.{DOMAIN}"], } -ROLES_WITH_THE_PROMETHEUS_SERVICE = ["web-app-nextcloud", "web-app-gitea"] +TLS_MODES = { + "web-app-nextcloud": "off", + "web-app-gitea": "self_signed", + "web-app-matrix": "letsencrypt", +} +ROLES_WITH_THE_PROMETHEUS_SERVICE = [ + "web-app-nextcloud", + "web-app-gitea", + "web-app-matrix", +] -def lookup(name, *terms, **_kwargs): - if name == "domains": - return DOMAINS - if name == "roles_with_service": - return [{"id": role} for role in ROLES_WITH_THE_PROMETHEUS_SERVICE] - if name == "tls": - term, want = terms - scheme = "http" if term.endswith(".onion") else "https" - return { - "url.base": f"{scheme}://{term}/", - "protocols.web": "http", - "enabled": False, - }[want] - if name == "domain": - return DOMAINS[terms[0]][0] - if name == "scrape_target": - return f"{terms[1]}:9000" - if name == "native_metrics_apps": - return [] - raise AssertionError(f"the template calls an unexpected lookup: {name!r}") +def probed_groups(*, ca_injected: bool) -> dict[str, dict]: + def lookup(name, *terms, **_kwargs): + if name == "domains": + return DOMAINS + if name == "roles_with_service": + return [{"id": role} for role in ROLES_WITH_THE_PROMETHEUS_SERVICE] + if name == "ca_injected": + return ca_injected + if name == "tls": + term, want = terms + if want == "mode": + return TLS_MODES[term] + scheme = "http" if term.endswith(".onion") else "https" + return { + "url.base": f"{scheme}://{term}/", + "protocols.web": "http", + "enabled": False, + }[want] + if name == "domain": + return DOMAINS[terms[0]][0] + if name == "scrape_target": + return f"{terms[1]}:9000" + if name == "native_metrics_apps": + return [] + raise AssertionError(f"the template calls an unexpected lookup: {name!r}") - -def probed_targets() -> dict[str, list[str]]: env = Environment( loader=FileSystemLoader(str(TEMPLATE_DIR)), undefined=StrictUndefined, @@ -60,11 +73,17 @@ def probed_targets() -> dict[str, list[str]]: ) jobs = {job["job_name"]: job for job in load_yaml_str(rendered)["scrape_configs"]} return { - group["labels"]["app"]: group["targets"] + group["labels"]["app"]: group for group in jobs["blackbox-healthz"]["static_configs"] } +def probed_targets() -> dict[str, list[str]]: + return { + app: group["targets"] for app, group in probed_groups(ca_injected=True).items() + } + + class TestBlackboxTargets(unittest.TestCase): def test_an_onion_app_is_probed_under_its_onion_name_over_http(self): self.assertEqual( @@ -81,6 +100,29 @@ class TestBlackboxTargets(unittest.TestCase): def test_only_the_roles_the_service_lookup_names_are_probed(self): self.assertEqual(set(probed_targets()), set(ROLES_WITH_THE_PROMETHEUS_SERVICE)) + def test_a_private_ca_target_skips_verification_while_the_ca_is_missing(self): + labels = probed_groups(ca_injected=False)["web-app-gitea"]["labels"] + self.assertEqual(labels["__param_module"], "http_ready_private_ca") + + def test_a_publicly_trusted_target_is_verified_while_the_ca_is_missing(self): + labels = probed_groups(ca_injected=False)["web-app-matrix"]["labels"] + self.assertNotIn("__param_module", labels) + + def test_a_plain_http_target_keeps_the_verifying_module(self): + labels = probed_groups(ca_injected=False)["web-app-nextcloud"]["labels"] + self.assertNotIn("__param_module", labels) + + def test_every_target_is_verified_once_the_ca_is_mounted(self): + groups = probed_groups(ca_injected=True) + self.assertEqual( + [ + app + for app, group in groups.items() + if "__param_module" in group["labels"] + ], + [], + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/python/scripts/lint/__init__.py b/tests/unit/python/scripts/lint/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/scripts/lint/test_mermaid.py b/tests/unit/python/scripts/lint/test_mermaid.py new file mode 100644 index 0000000000..67d3099a2f --- /dev/null +++ b/tests/unit/python/scripts/lint/test_mermaid.py @@ -0,0 +1,62 @@ +"""``scripts/lint/mermaid.sh`` renders with the browser install-lint provisioned. + +``make lint`` runs its targets in parallel, so the script must not reach for a +package manager, and a browser that cannot start must fail the lint once with +its reason instead of once per Markdown file. +""" + +from __future__ import annotations + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = Path(PROJECT_ROOT) / "scripts" / "lint" / "mermaid.sh" +LIBRARY_ERROR = ( + "chrome-headless-shell: error while loading shared libraries: libnspr4.so" +) +RECORDING_STUB = '#!/bin/sh\necho "$(basename "$0")" >> "$STUB_CALLS"\n' + + +class TestMermaidLint(unittest.TestCase): + def test_a_browser_that_cannot_start_fails_before_any_file_renders(self): + with tempfile.TemporaryDirectory() as tmp: + bin_dir = Path(tmp) + calls = bin_dir / "calls" + calls.touch() + for name in ("npx", "apt-get"): + (bin_dir / name).write_text(RECORDING_STUB) + (bin_dir / "mmdc").write_text( + f"{RECORDING_STUB}echo '{LIBRARY_ERROR}' >&2\nexit 1\n" + ) + for stub in bin_dir.iterdir(): + stub.chmod(0o755) + env = { + k: v + for k, v in os.environ.items() + if not k.startswith("INFINITO_") and k != "BASH_ENV" + } + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + env["STUB_CALLS"] = str(calls) + proc = subprocess.run( + ["bash", str(SCRIPT)], + env=env, + capture_output=True, + text=True, + check=False, + ) + recorded = calls.read_text().split() # nocheck: cache-read stub call log + + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertIn("cannot render a minimal diagram", proc.stderr) + self.assertIn(LIBRARY_ERROR, proc.stderr) + self.assertNotIn("❌", proc.stdout) + self.assertEqual(recorded, ["mmdc"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/meta/env/__init__.py b/tests/unit/python/scripts/meta/env/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/scripts/meta/env/test_load.py b/tests/unit/python/scripts/meta/env/test_load.py new file mode 100644 index 0000000000..1fa719719d --- /dev/null +++ b/tests/unit/python/scripts/meta/env/test_load.py @@ -0,0 +1,136 @@ +"""The distro-derived keys follow the distro the caller asks for. + +``scripts/tests/deploy/distros.sh`` exports ``INFINITO_DISTRO`` per iteration +and sources the loader again. The keys composed from the distro must then name +that distro, not the one ``.env`` was generated for, and ``.env`` itself must +stay as it was. +""" + +from __future__ import annotations + +import filecmp +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import ClassVar + +from utils.cache.files import PROJECT_ROOT + +LOADER_DIR = Path(PROJECT_ROOT) / "scripts" / "meta" / "env" +SHOWN = ( + "INFINITO_DISTRO", + "INFINITO_CONTAINER", + "INFINITO_PARENT_IMAGE", + "INFINITO_IMAGE", +) +SHOW = "printf '%s\\n' " + " ".join(f'"{key}=${{{key}:-}}"' for key in SHOWN) + + +def clean_env(**overrides: str) -> dict[str, str]: + env = { + k: v + for k, v in os.environ.items() + if not k.startswith("INFINITO_") + and k not in ("BASH_ENV", "PYTHONPATH", "VENV", "PYTHON", "VIRTUAL_ENV") + } + env["PYTHONPATH"] = str(PROJECT_ROOT) + env.update(overrides) + return env + + +def bash(script: str, cwd: Path) -> dict[str, str]: + proc = subprocess.run( + ["bash", "-c", f"{script}\n{SHOW}"], + env=clean_env(), + cwd=cwd, + capture_output=True, + text=True, + check=True, + ) + return dict(line.split("=", 1) for line in proc.stdout.splitlines()) + + +class TestDistroDerivedKeys(unittest.TestCase): + scratch: ClassVar[Path] + expected: ClassVar[dict[str, dict[str, str]]] + + @classmethod + def setUpClass(cls) -> None: + cls.scratch = Path(tempfile.mkdtemp()) + cls.expected = {} + for distro in ("debian", "centos"): + path = cls.scratch / f"{distro}.env" + subprocess.run( + [sys.executable, "-m", "cli.meta.env", "--output", str(path)], + env=clean_env(INFINITO_DISTRO=distro), + cwd=PROJECT_ROOT, + capture_output=True, + check=True, + ) + cls.expected[distro] = bash(f'set -a; source "{path}"', cls.scratch) + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.scratch) + + def run_loader(self, steps: str) -> dict[str, str]: + """Source a copy of the loader in a checkout whose ``.env`` is debian's. + + Args: + steps: bash run before the keys are shown; ``load`` sources the + loader. + + Returns: + The keys as the steps leave them. + """ + checkout = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, checkout) + (checkout / "scripts" / "meta" / "env").mkdir(parents=True) + for name in ("load.sh", "python.sh"): + shutil.copy(LOADER_DIR / name, checkout / "scripts" / "meta" / "env") + dotenv = checkout / ".env" + shutil.copy(self.scratch / "debian.env", dotenv) + shown = bash( + f"load() {{ source scripts/meta/env/load.sh; }}\n{steps}", checkout + ) + self.assertTrue( + filecmp.cmp(dotenv, self.scratch / "debian.env", shallow=False), + ".env was rewritten", + ) + return shown + + def test_the_two_distros_derive_different_keys(self): + self.assertNotEqual( + self.expected["debian"]["INFINITO_PARENT_IMAGE"], + self.expected["centos"]["INFINITO_PARENT_IMAGE"], + ) + + def test_a_distro_exported_after_the_first_load_gets_its_own_keys(self): + shown = self.run_loader("load\nexport INFINITO_DISTRO=centos\nload") + self.assertEqual(shown, self.expected["centos"]) + + def test_a_distro_exported_before_the_first_load_gets_its_own_keys(self): + shown = self.run_loader("export INFINITO_DISTRO=centos\nload") + self.assertEqual(shown, self.expected["centos"]) + + def test_switching_back_restores_the_keys_of_the_dotenv(self): + shown = self.run_loader( + "load\nexport INFINITO_DISTRO=centos\nload\n" + "export INFINITO_DISTRO=debian\nload" + ) + self.assertEqual(shown, self.expected["debian"]) + + def test_a_caller_value_for_the_loaded_distro_still_wins(self): + shown = self.run_loader( + "load\nexport INFINITO_PARENT_IMAGE=caller/image:pinned\n" + "touch -d '+1 minute' .env\nload" + ) + self.assertEqual(shown["INFINITO_PARENT_IMAGE"], "caller/image:pinned") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/meta/resolve/missing/__init__.py b/tests/unit/python/scripts/meta/resolve/missing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/scripts/meta/resolve/missing/test_highest_release_tag.py b/tests/unit/python/scripts/meta/resolve/missing/test_highest_release_tag.py new file mode 100644 index 0000000000..c9801bb119 --- /dev/null +++ b/tests/unit/python/scripts/meta/resolve/missing/test_highest_release_tag.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = ( + PROJECT_ROOT / "scripts" / "meta" / "resolve" / "missing" / "highest_release_tag.sh" +) +TAGS = ("v9.0.0", "v9.0.1", "v9.0.2", "v14.1.0", "v15.0.0") +STUBS = { + "git": '#!/usr/bin/env bash\nprintf "%s\\n" ${STUB_TAGS}\n', + "docker": ( + "#!/usr/bin/env bash\n" + 'case " ${STUB_MISSING} " in *" ${3##*:} "*) exit 1 ;; esac\n' + ), +} + + +def pick(*missing: str) -> subprocess.CompletedProcess: + """Run the picker against stubbed ``git`` tags and ``docker`` manifests. + + Args: + missing: tags whose images the registry does not hold. + """ + with tempfile.TemporaryDirectory() as tmp: + for name, body in STUBS.items(): + stub = Path(tmp) / name + stub.write_text(body) + stub.chmod(0o755) + env = { + **{k: v for k, v in os.environ.items() if k != "BASH_ENV"}, + "PATH": f"{tmp}{os.pathsep}{os.environ['PATH']}", + "STUB_TAGS": " ".join(TAGS), + "STUB_MISSING": " ".join(missing), + "OWNER": "acme", + "REGISTRY": "ghcr.io", + "REPO_PREFIX": "core", + "INFINITO_DISTROS": "debian arch", + } + return subprocess.run( + ["bash", str(SCRIPT)], + env=env, + capture_output=True, + text=True, + check=True, + ) + + +class TestHighestReleaseTag(unittest.TestCase): + def test_tags_below_the_oldest_buildable_one_are_not_picked(self) -> None: + self.assertEqual("", pick("v9.0.1", "v9.0.0").stdout) + + def test_the_oldest_buildable_tag_is_picked(self) -> None: + self.assertEqual("v9.0.2\n", pick("v9.0.2", "v9.0.1").stdout) + + def test_the_highest_missing_tag_is_picked(self) -> None: + self.assertEqual("v14.1.0\n", pick("v14.1.0", "v9.0.1").stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/tests/deploy/act/__init__.py b/tests/unit/python/scripts/tests/deploy/act/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/scripts/tests/deploy/act/test_down_act_outer.py b/tests/unit/python/scripts/tests/deploy/act/test_down_act_outer.py new file mode 100644 index 0000000000..8082bf8f20 --- /dev/null +++ b/tests/unit/python/scripts/tests/deploy/act/test_down_act_outer.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "scripts" / "tests" / "deploy" / "act" / "down_act_outer.sh" +DOCKER_STUB = ( + "#!/usr/bin/env bash\n" + 'printf "%s\\n" "$*" >>"${STUB_LOG}"\n' + 'if [ "$1" = ps ]; then\n' + ' case "$*" in *"label=infinito.worktree=own"*) echo own-id ;; esac\n' + "fi\n" +) + + +def teardown(container: str | None) -> tuple[subprocess.CompletedProcess, list[str]]: + """Run the teardown against a ``docker`` stub that only knows ``own-id``. + + Args: + container: value of ``INFINITO_CONTAINER``, or None to leave it unset. + + Returns: + The finished process and the docker invocations it made. + """ + with tempfile.TemporaryDirectory() as tmp: + stub = Path(tmp) / "docker" + stub.write_text(DOCKER_STUB) + stub.chmod(0o755) + log = Path(tmp) / "docker.log" + log.touch() + env = { + k: v + for k, v in os.environ.items() + if k not in ("BASH_ENV", "INFINITO_CONTAINER") + } + env.update(PATH=f"{tmp}{os.pathsep}{os.environ['PATH']}", STUB_LOG=str(log)) + if container is not None: + env["INFINITO_CONTAINER"] = container + done = subprocess.run( + ["bash", str(SCRIPT)], + env=env, + capture_output=True, + text=True, + check=False, + ) + return done, log.read_text().splitlines() # nocheck: cache-read stub call log + + +class TestDownActOuter(unittest.TestCase): + def test_only_the_own_worktrees_act_containers_are_listed(self) -> None: + _, calls = teardown("own") + self.assertIn("label=infinito.worktree=own", calls[0]) + + def test_the_own_act_container_is_removed(self) -> None: + _, calls = teardown("own") + self.assertEqual(calls[1], "rm -f own-id") + + def test_a_foreign_worktrees_act_container_is_left_alone(self) -> None: + _, calls = teardown("foreign") + self.assertEqual(len(calls), 1) + + def test_an_unset_container_name_removes_nothing(self) -> None: + done, calls = teardown(None) + self.assertNotEqual(done.returncode, 0) + self.assertEqual(calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/tests/deploy/test_distros.py b/tests/unit/python/scripts/tests/deploy/test_distros.py index 4b6cb76aee..80c4d7903f 100644 --- a/tests/unit/python/scripts/tests/deploy/test_distros.py +++ b/tests/unit/python/scripts/tests/deploy/test_distros.py @@ -20,6 +20,7 @@ import unittest from pathlib import Path from utils.cache.files import PROJECT_ROOT, read_text +from utils.distros import pkgmgr_image from utils.env.parser import parse_static_env LOOP = PROJECT_ROOT / "scripts" / "tests" / "deploy" / "distros.sh" @@ -76,6 +77,19 @@ class TestDistrosLoop(unittest.TestCase): self.assertEqual(proc.returncode, 0, proc.stderr) self.assertEqual(sorted(record.split()), sorted(DISTROS.split())) + def test_every_distro_runs_with_a_parent_image_of_its_own(self) -> None: + proc, record = self._run( + DISTROS, + 'echo "${INFINITO_DISTRO} ${INFINITO_PARENT_IMAGE}' + ' ${INFINITO_PARENT_IMAGE_OWNER} ${INFINITO_PARENT_IMAGE_TAG}" >> "${RECORD}"', + INFINITO_CI_DISTRO_BUDGET_SECONDS="600", + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + rows = [line.split() for line in record.splitlines()] + self.assertEqual(sorted(row[0] for row in rows), sorted(DISTROS.split())) + for distro, image, owner, tag in rows: + self.assertEqual(image, pkgmgr_image(distro, owner=owner, tag=tag)) + def test_the_command_inherits_the_env_layer(self) -> None: proc, record = self._run( "debian", diff --git a/tests/unit/python/utils/docker/test_image_discovery_lifecycle.py b/tests/unit/python/utils/docker/test_image_discovery_lifecycle.py new file mode 100644 index 0000000000..5c7386125a --- /dev/null +++ b/tests/unit/python/utils/docker/test_image_discovery_lifecycle.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import os +import tempfile +import unittest +from pathlib import Path + +from utils.docker.image.discovery import iter_role_images +from utils.roles.mapping import ROLE_FILE_META_SERVICES + +SUPPORTED = """--- +demo: + image: libretranslate/libretranslate + version: v1.9.6 + lifecycle: stable +helper: + image: alpine + version: "3.22" +""" + +RETIRED = """--- +demo: + image: quay.io/minio/minio + version: RELEASE.2025-09-07T16-13-09Z + lifecycle: eol +client: + image: quay.io/minio/mc + version: RELEASE.2025-08-13T08-35-41Z +""" + +UNSTAGED = """--- +demo: + image: libretranslate/libretranslate + version: v1.9.6 +""" + + +class TestImageDiscoveryLifecycle(unittest.TestCase): + def _images(self, services_yaml: str, *, relative: bool = False) -> set[str]: + """Discover the images of one role written into a temporary repository. + + Args: + services_yaml: the role's ``meta/services.yml``. + relative: pass the repository root relative to the working directory. + """ + with tempfile.TemporaryDirectory() as tmp: + services = Path(tmp) / "roles" / "web-app-demo" / ROLE_FILE_META_SERVICES + services.parent.mkdir(parents=True) + services.write_text(services_yaml, encoding="utf-8") + root = Path(os.path.relpath(tmp)) if relative else Path(tmp) + return {ref.service for ref in iter_role_images(root)} + + def test_a_supported_role_yields_every_service(self) -> None: + self.assertEqual({"demo", "helper"}, self._images(SUPPORTED)) + + def test_a_retired_role_yields_none_of_its_services(self) -> None: + self.assertEqual(set(), self._images(RETIRED)) + + def test_a_role_that_names_no_stage_is_kept(self) -> None: + self.assertEqual({"demo"}, self._images(UNSTAGED)) + + def test_a_relative_root_still_leaves_a_retired_role_out(self) -> None: + self.assertEqual(set(), self._images(RETIRED, relative=True)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/github/test_playwright_summary.py b/tests/unit/python/utils/github/test_playwright_summary.py index 25743f31b7..36b0ebb07f 100644 --- a/tests/unit/python/utils/github/test_playwright_summary.py +++ b/tests/unit/python/utils/github/test_playwright_summary.py @@ -10,7 +10,7 @@ directories / no junit files / unparseable junit files. from __future__ import annotations -import datetime +import datetime as dt import io import os import unittest @@ -56,9 +56,7 @@ def _write(path: Path, content: str) -> Path: class TestToIsoZ(unittest.TestCase): def test_utc_datetime_formats_with_z_suffix(self) -> None: self.assertEqual( - ps._to_iso_z( - datetime.datetime(2026, 5, 17, 12, 34, 56, tzinfo=datetime.UTC) - ), + ps._to_iso_z(dt.datetime(2026, 5, 17, 12, 34, 56, tzinfo=dt.UTC)), "2026-05-17T12:34:56Z", ) @@ -263,9 +261,7 @@ class TestSummarizeFile(unittest.TestCase): Path(tmp) / "web-app-foo" / "playwright-junit.xml", no_timestamp_fixture, ) - target = datetime.datetime( - 2025, 5, 17, 5, 34, 56, tzinfo=datetime.UTC - ).timestamp() + target = dt.datetime(2025, 5, 17, 5, 34, 56, tzinfo=dt.UTC).timestamp() os.utime(xml_path, (target, target)) summary = ps._summarize_file(xml_path) assert summary is not None diff --git a/tests/unit/python/utils/install/lint/test_main.py b/tests/unit/python/utils/install/lint/test_main.py index c044c3afa9..1ac3539ca8 100644 --- a/tests/unit/python/utils/install/lint/test_main.py +++ b/tests/unit/python/utils/install/lint/test_main.py @@ -34,12 +34,30 @@ class TestStampLogic(unittest.TestCase): with ( mock.patch.object(cli, "PROJECT_ROOT", root), mock.patch.object(cli.shutil, "which", return_value="/usr/bin/x"), + mock.patch.object( + cli.mermaid_cli, "browser_installed", return_value=True + ), mock.patch.object(cli, "_install_all") as inst, ): rc = cli.main([]) self.assertEqual(rc, 0) inst.assert_not_called() + def test_missing_mermaid_browser_invalidates_stamp(self) -> None: + with TemporaryDirectory() as tmp: + root = _make_repo_root(tmp, stamp_age_offset=60.0) + with ( + mock.patch.object(cli, "PROJECT_ROOT", root), + mock.patch.object(cli.shutil, "which", return_value="/usr/bin/x"), + mock.patch.object( + cli.mermaid_cli, "browser_installed", return_value=False + ), + mock.patch.object(cli, "_install_all") as inst, + ): + rc = cli.main([]) + self.assertEqual(rc, 0) + inst.assert_called_once() + def test_missing_tool_invalidates_stamp(self) -> None: with TemporaryDirectory() as tmp: root = _make_repo_root(tmp, stamp_age_offset=60.0) @@ -81,9 +99,10 @@ class TestStampLogic(unittest.TestCase): self.assertTrue((root / cli._STAMP).exists()) def test_force_drops_stamp(self) -> None: - with TemporaryDirectory() as tmp: + with TemporaryDirectory() as tmp, TemporaryDirectory() as cache: root = _make_repo_root(tmp, stamp_age_offset=60.0) with ( + mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}), mock.patch.object(cli, "PROJECT_ROOT", root), mock.patch.object(cli, "_install_all") as inst, ): @@ -91,6 +110,19 @@ class TestStampLogic(unittest.TestCase): self.assertEqual(rc, 0) inst.assert_called_once() + def test_force_drops_the_mermaid_browser(self) -> None: + with TemporaryDirectory() as tmp, TemporaryDirectory() as cache: + root = _make_repo_root(tmp, stamp_age_offset=60.0) + browser = Path(cache) / "chrome-headless-shell" / "linux-1" + browser.mkdir(parents=True) + with ( + mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}), + mock.patch.object(cli, "PROJECT_ROOT", root), + mock.patch.object(cli, "_install_all"), + ): + cli.main(["--force"]) + self.assertFalse(browser.exists()) + class TestGroupDispatch(unittest.TestCase): def test_group_only_bypasses_stamp(self) -> None: diff --git a/tests/unit/python/utils/install/lint/test_mermaid_cli.py b/tests/unit/python/utils/install/lint/test_mermaid_cli.py new file mode 100644 index 0000000000..5b66791177 --- /dev/null +++ b/tests/unit/python/utils/install/lint/test_mermaid_cli.py @@ -0,0 +1,228 @@ +"""Unit tests for :mod:`utils.install.lint.mermaid_cli`.""" + +from __future__ import annotations + +import contextlib +import os +import subprocess +import unittest +import unittest.mock as mock +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING + +from utils.install.lint import mermaid_cli + +if TYPE_CHECKING: + from collections.abc import Iterator + +REVISION = "155.0.8059.39" + + +def _done(returncode: int) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess([], returncode, stdout="out\n", stderr="err\n") + + +def _write_core(node_modules: Path, revision: str) -> None: + revisions = node_modules / "puppeteer-core" / "lib" / "puppeteer" + revisions.mkdir(parents=True) + (revisions / "revisions.js").write_text( + f"export const PUPPETEER_REVISIONS = Object.freeze({{\n" + f" chrome: '155.0.8059.39',\n" + f" 'chrome-headless-shell': '{revision}',\n" + f"}});\n", + encoding="utf-8", + ) + + +def _fake_mmdc( + root: Path, *, revision: str | None = REVISION, nested: str | None = None +) -> Path: + """Lay out a global mermaid-cli install the way npm does. + + Args: + root: npm prefix to build the install under. + revision: chrome-headless-shell revision of the puppeteer-core beside + puppeteer, or None to install mmdc without a puppeteer next to it. + nested: revision of a puppeteer-core nested under puppeteer, as npm + lays it out when mermaid-cli pins another puppeteer-core. + + Returns: + The ``bin/mmdc`` symlink. + """ + package = root / "lib" / "node_modules" / "@mermaid-js" / "mermaid-cli" + (package / "src").mkdir(parents=True) + (package / "src" / "cli.js").write_text("", encoding="utf-8") + if revision is not None: + (package / "node_modules" / "puppeteer").mkdir(parents=True) + _write_core(package / "node_modules", revision) + if nested is not None: + _write_core(package / "node_modules" / "puppeteer" / "node_modules", nested) + (root / "bin").mkdir() + mmdc = root / "bin" / "mmdc" + mmdc.symlink_to(package / "src" / "cli.js") + return mmdc + + +def _install_browser(cache: Path, revision: str) -> None: + build = cache / "chrome-headless-shell" / f"linux-{revision}" / "linux64" + build.mkdir(parents=True) + (build / "chrome-headless-shell").write_text("", encoding="utf-8") + + +@contextlib.contextmanager +def _environment( + *, revision: str | None = REVISION, nested: str | None = None +) -> Iterator[tuple[Path, Path]]: + """Patch mmdc's location and the puppeteer cache onto temporary dirs. + + Args: + revision: passed to :func:`_fake_mmdc`. + nested: passed to :func:`_fake_mmdc`. + + Yields: + The npm prefix and the puppeteer cache directory. + """ + with TemporaryDirectory() as prefix, TemporaryDirectory() as cache: + mmdc = _fake_mmdc(Path(prefix), revision=revision, nested=nested) + with ( + mock.patch.dict(os.environ, {"PUPPETEER_CACHE_DIR": cache}), + mock.patch.object(mermaid_cli, "which", return_value=True), + mock.patch.object(mermaid_cli.shutil, "which", return_value=str(mmdc)), + ): + yield Path(prefix), Path(cache) + + +class TestBrowserProvisioning(unittest.TestCase): + def provision(self, *, euid: int, results: list[int]) -> tuple[list, list[str]]: + """Run ``ensure()`` with mmdc present and a stubbed puppeteer. + + Args: + euid: effective uid the run pretends to have. + results: exit codes the successive puppeteer calls return. + + Returns: + The recorded subprocess calls and the warnings emitted. + """ + warnings: list[str] = [] + with ( + _environment(), + mock.patch.object(mermaid_cli.os, "geteuid", return_value=euid), + mock.patch.object(mermaid_cli, "warn", side_effect=warnings.append), + mock.patch.object( + mermaid_cli.subprocess, + "run", + side_effect=[_done(code) for code in results], + ) as run, + ): + mermaid_cli.ensure() + return run.call_args_list, warnings + + def test_root_installs_the_browser_with_its_libraries(self) -> None: + calls, warnings = self.provision(euid=0, results=[0]) + self.assertEqual(len(calls), 1) + self.assertEqual( + calls[0].args[0][-2:], ["chrome-headless-shell", "--install-deps"] + ) + self.assertEqual(warnings, []) + + def test_the_browser_comes_from_the_puppeteer_next_to_mmdc(self) -> None: + calls, _ = self.provision(euid=0, results=[0]) + self.assertTrue( + calls[0] + .args[0][0] + .endswith("@mermaid-js/mermaid-cli/node_modules/.bin/puppeteer") + ) + + def test_a_user_installs_the_browser_without_libraries(self) -> None: + calls, _ = self.provision(euid=1000, results=[0]) + self.assertNotIn("--install-deps", calls[0].args[0]) + + def test_the_browser_install_downloads_despite_the_mmdc_skip_flag(self) -> None: + with mock.patch.dict(os.environ, {"PUPPETEER_SKIP_DOWNLOAD": "1"}): + calls, _ = self.provision(euid=0, results=[0]) + self.assertNotIn("PUPPETEER_SKIP_DOWNLOAD", calls[0].kwargs["env"]) + + def test_a_failed_install_clears_its_revision_and_retries_once(self) -> None: + seen: list[tuple[bool, bool]] = [] + with _environment() as (_, cache): + partial = cache / "chrome-headless-shell" / f"linux-{REVISION}" + partial.mkdir(parents=True) + other = cache / "chrome-headless-shell" / "linux-155.0.8059.79" + other.mkdir(parents=True) + + def run(*_args, **_kwargs): + seen.append((partial.exists(), other.exists())) + return _done(1 if len(seen) == 1 else 0) + + with mock.patch.object(mermaid_cli.subprocess, "run", side_effect=run): + mermaid_cli.ensure() + self.assertEqual(seen, [(True, True), (False, True)]) + + def test_a_second_failure_warns_with_puppeteers_output(self) -> None: + calls, warnings = self.provision(euid=0, results=[1, 1]) + self.assertEqual(len(calls), 2) + self.assertEqual(len(warnings), 1) + self.assertIn("out\nerr\n", warnings[0]) + + def test_the_pinned_browser_is_not_provisioned_again(self) -> None: + with _environment() as (_, cache): + _install_browser(cache, REVISION) + with mock.patch.object(mermaid_cli.subprocess, "run") as run: + mermaid_cli.ensure() + run.assert_not_called() + + def test_a_browser_of_another_revision_is_replaced(self) -> None: + with _environment() as (_, cache): + _install_browser(cache, "155.0.8059.79") + with mock.patch.object( + mermaid_cli.subprocess, "run", return_value=_done(0) + ) as run: + mermaid_cli.ensure() + run.assert_called_once() + + def test_mmdc_without_its_own_puppeteer_warns(self) -> None: + with ( + _environment(revision=None), + mock.patch.object(mermaid_cli, "warn") as warn, + mock.patch.object(mermaid_cli.subprocess, "run") as run, + ): + mermaid_cli.ensure() + run.assert_not_called() + warn.assert_called_once() + + def test_a_timeout_warns_instead_of_raising(self) -> None: + with ( + _environment(), + mock.patch.object( + mermaid_cli.subprocess, + "run", + side_effect=subprocess.TimeoutExpired("puppeteer", 900), + ), + mock.patch.object(mermaid_cli, "warn") as warn, + ): + mermaid_cli.ensure() + warn.assert_called_once() + + +class TestBrowserInstalled(unittest.TestCase): + def test_only_the_pinned_revision_counts(self) -> None: + with _environment() as (_, cache): + _install_browser(cache, "155.0.8059.79") + self.assertFalse(mermaid_cli.browser_installed()) + _install_browser(cache, REVISION) + self.assertTrue(mermaid_cli.browser_installed()) + + def test_the_revision_is_read_from_mmdcs_puppeteer_core(self) -> None: + with _environment(): + self.assertEqual(mermaid_cli.browser_revision(), REVISION) + + def test_a_puppeteer_core_nested_under_puppeteer_wins(self) -> None: + with _environment(revision="155.0.8059.79", nested=REVISION) as (_, cache): + self.assertEqual(mermaid_cli.browser_revision(), REVISION) + _install_browser(cache, REVISION) + self.assertTrue(mermaid_cli.browser_installed()) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/docker/image/discovery.py b/utils/docker/image/discovery.py index 3246b8fdd5..7f1c7bac0d 100644 --- a/utils/docker/image/discovery.py +++ b/utils/docker/image/discovery.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING from utils.cache.yaml import load_yaml_any as _load_yaml_cached from utils.docker.image.pin import pull_reference from utils.roles.mapping import ROLE_FILE_META_SERVICES +from utils.update.base import is_maintained if TYPE_CHECKING: from collections.abc import Iterable @@ -191,14 +192,18 @@ def iter_role_images(repo_root: Path) -> Iterable[ImageRef]: """ Yield all ImageRef entries discovered across all roles in *repo_root*. - Source: roles/**/meta/services.yml → .{image,version}. + Source: roles/**/meta/services.yml → .{image,version}. Roles the + updaters do not maintain (``is_maintained``) are left out. See docs/contributing/artefact/image.md for the full format reference. """ - roles_dir = repo_root / "roles" + roles_dir = (repo_root / "roles").resolve() for services_file in roles_dir.glob(f"**/{ROLE_FILE_META_SERVICES}"): role_name = services_file.parent.parent.name + if not is_maintained(roles_dir, role_name): + continue + services = load_yaml(services_file) if not isinstance(services, dict): diff --git a/utils/github/playwright_summary.py b/utils/github/playwright_summary.py index e448f531df..ef33a84d75 100644 --- a/utils/github/playwright_summary.py +++ b/utils/github/playwright_summary.py @@ -12,7 +12,7 @@ Usage: from __future__ import annotations import contextlib -import datetime +import datetime as dt import sys import xml.etree.ElementTree as ET from pathlib import Path @@ -29,12 +29,12 @@ _STATUS_EMOJI: dict[str, str] = { } -def _to_iso_z(dt: datetime.datetime) -> str: - """Format ``dt`` as ``YYYY-MM-DDTHH:MM:SSZ`` (UTC, second precision).""" - return dt.astimezone(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ") +def _to_iso_z(moment: dt.datetime) -> str: + """Format ``moment`` as ``YYYY-MM-DDTHH:MM:SSZ`` (UTC, second precision).""" + return moment.astimezone(dt.UTC).strftime("%Y-%m-%dT%H:%M:%SZ") -def _parse_iso_timestamp(value: str | None) -> datetime.datetime | None: +def _parse_iso_timestamp(value: str | None) -> dt.datetime | None: """Parse a JUnit ``timestamp`` attribute. Treat tz-naive values as UTC. Returns ``None`` when the value is empty or unparseable so callers @@ -43,17 +43,17 @@ def _parse_iso_timestamp(value: str | None) -> datetime.datetime | None: if not value: return None try: - dt = datetime.datetime.fromisoformat(value) + moment = dt.datetime.fromisoformat(value) except ValueError: return None - if dt.tzinfo is None: - dt = dt.replace(tzinfo=datetime.UTC) - return dt + if moment.tzinfo is None: + moment = moment.replace(tzinfo=dt.UTC) + return moment -def _file_mtime_utc(path: Path) -> datetime.datetime: +def _file_mtime_utc(path: Path) -> dt.datetime: """Last-modified time of *path* as a UTC ``datetime``.""" - return datetime.datetime.fromtimestamp(path.stat().st_mtime, tz=datetime.UTC) + return dt.datetime.fromtimestamp(path.stat().st_mtime, tz=dt.UTC) def _format_duration(seconds: float) -> str: @@ -153,7 +153,7 @@ def _summarize_file(path: Path) -> dict | None: base_records = _extract_records(cases) offset_seconds = 0.0 for name, status, msg, duration in base_records: - case_start = suite_ts + datetime.timedelta(seconds=offset_seconds) + case_start = suite_ts + dt.timedelta(seconds=offset_seconds) enriched.append((_to_iso_z(case_start), name, status, msg, duration)) offset_seconds += duration if not enriched: diff --git a/utils/install/lint/__main__.py b/utils/install/lint/__main__.py index 11fb1fc56a..9fed403e4b 100644 --- a/utils/install/lint/__main__.py +++ b/utils/install/lint/__main__.py @@ -2,7 +2,8 @@ Mirrors the prior ``scripts/install/lint.sh`` interface: no-arg / ``all`` runs every group + manages the stamp file; explicit groups bypass the -stamp; ``--force`` (as first arg) drops the stamp and reinstalls. +stamp; ``--force`` (as first arg) drops the stamp and mmdc's browser cache +and reinstalls. """ from __future__ import annotations @@ -177,7 +178,10 @@ def _stamp_is_fresh(repo_root: Path) -> bool: return False if dep_path.stat().st_mtime > stamp_mtime: return False - return all(shutil.which(tool) is not None for tool in _STAMP_TOOLS) + return ( + all(shutil.which(tool) is not None for tool in _STAMP_TOOLS) + and mermaid_cli.browser_installed() + ) def _touch_stamp(repo_root: Path) -> None: @@ -212,6 +216,7 @@ def main(argv: list[str] | None = None) -> int: stamp_path = repo_root / _STAMP if stamp_path.exists(): stamp_path.unlink() + shutil.rmtree(mermaid_cli.browser_cache(), ignore_errors=True) args = args[1:] if not args: diff --git a/utils/install/lint/mermaid_cli.py b/utils/install/lint/mermaid_cli.py index 099af21a4d..f1782e0d3a 100644 --- a/utils/install/lint/mermaid_cli.py +++ b/utils/install/lint/mermaid_cli.py @@ -1,24 +1,30 @@ -"""Install @mermaid-js/mermaid-cli (mmdc) via npm global. +"""Install @mermaid-js/mermaid-cli (mmdc) and the chrome-headless-shell it renders with. -Best-effort: the puppeteer browser is not downloaded here -(``PUPPETEER_SKIP_DOWNLOAD``) so ``install-lint`` stays sandbox-safe. mmdc only -needs a browser to RENDER, which ``scripts/lint/mermaid.sh`` provisions on demand -(``npx puppeteer browsers install chrome-headless-shell``) right before it -renders. That browser ships as a zip, so ``unzip`` is ensured here (puppeteer's -extractor needs it, else it leaves an empty version dir and mmdc reports the -browser missing). A failed install warns instead of aborting ``install-lint``, -so sandboxes without npm/browser access still get the rest of the lint toolchain. +mmdc comes from npm with ``PUPPETEER_SKIP_DOWNLOAD`` set; the browser is +provisioned afterwards through the puppeteer that npm installed next to mmdc, +at the revision its puppeteer-core pins, as root with its shared libraries +(``--install-deps``). ``unzip`` is ensured first because puppeteer extracts the +browser with it. A failed install warns instead of aborting ``install-lint``, so +sandboxes without npm or browser access still get the rest of the lint +toolchain. """ from __future__ import annotations import contextlib import os +import re +import shutil +import subprocess +from pathlib import Path from utils.install.npm import npm_install_global from utils.install.primitives import log, warn, which from utils.install.system_pkg import install_command_via_pkg +BROWSER = "chrome-headless-shell" +PROVISION_TIMEOUT_S = 900 + def _ensure_unzip() -> None: if which("unzip"): @@ -27,16 +33,114 @@ def _ensure_unzip() -> None: install_command_via_pkg("unzip") +def browser_cache() -> Path: + base = os.environ.get("PUPPETEER_CACHE_DIR") or Path.home() / ".cache" / "puppeteer" + return Path(base) / BROWSER + + +def _mmdc_puppeteer() -> Path | None: + mmdc = shutil.which("mmdc") + if mmdc is None: + return None + for parent in Path(mmdc).resolve().parents: + puppeteer = parent / "node_modules" / "puppeteer" + if puppeteer.is_dir(): + return puppeteer + return None + + +def _puppeteer_core(puppeteer: Path) -> Path | None: + for base in (puppeteer, *puppeteer.parents): + if base.name == "node_modules": + continue + core = base / "node_modules" / "puppeteer-core" + if core.is_dir(): + return core + return None + + +def browser_revision() -> str | None: + puppeteer = _mmdc_puppeteer() + if puppeteer is None: + return None + core = _puppeteer_core(puppeteer) + if core is None: + return None + revisions = core / "lib" / "puppeteer" / "revisions.js" + try: + text = revisions.read_text(encoding="utf-8") # nocheck: cache-read npm prefix + except OSError: + return None + match = re.search(rf"'{BROWSER}':\s*'([^']+)'", text) + return match.group(1) if match else None + + +def browser_installed() -> bool: + revision = browser_revision() + if revision is None: + return False + return any( + path.is_file() for path in browser_cache().glob(f"*-{revision}/*/{BROWSER}") + ) + + +def _provision_browser(puppeteer: Path) -> subprocess.CompletedProcess[str]: + argv = [ + str(puppeteer.parent / ".bin" / "puppeteer"), + "browsers", + "install", + BROWSER, + ] + if os.geteuid() == 0: + argv.append("--install-deps") + env = {k: v for k, v in os.environ.items() if k != "PUPPETEER_SKIP_DOWNLOAD"} + return subprocess.run( + argv, + env=env, + capture_output=True, + text=True, + timeout=PROVISION_TIMEOUT_S, + check=False, + ) + + +def _ensure_browser() -> None: + if browser_installed(): + return + puppeteer = _mmdc_puppeteer() + if puppeteer is None: + warn(f"{BROWSER} provisioning skipped: no puppeteer next to mmdc") + return + log(f"Provisioning {BROWSER} for mmdc via its own puppeteer.") + try: + done = _provision_browser(puppeteer) + if done.returncode != 0: + revision = browser_revision() + if revision is not None: + for partial in browser_cache().glob(f"*-{revision}"): + shutil.rmtree(partial, ignore_errors=True) + done = _provision_browser(puppeteer) + except (OSError, subprocess.TimeoutExpired) as exc: + warn(f"{BROWSER} provisioning skipped (optional lint tool): {exc}") + return + if done.returncode != 0: + warn( + f"{BROWSER} provisioning failed (optional lint tool):\n" + f"{done.stdout}{done.stderr}" + ) + + def ensure() -> None: _ensure_unzip() - if which("mmdc"): - return - log("Missing command 'mmdc'. Installing @mermaid-js/mermaid-cli via npm.") - os.environ["PUPPETEER_SKIP_DOWNLOAD"] = "1" - try: - npm_install_global("@mermaid-js/mermaid-cli") - except Exception as exc: # noqa: BLE001 - best-effort optional lint tool - warn(f"mermaid-cli install skipped (optional lint tool): {exc}") - return if not which("mmdc"): - warn("Command 'mmdc' is still unavailable after installation.") + log("Missing command 'mmdc'. Installing @mermaid-js/mermaid-cli via npm.") + os.environ["PUPPETEER_SKIP_DOWNLOAD"] = "1" + try: + npm_install_global("@mermaid-js/mermaid-cli") + except Exception as exc: # noqa: BLE001 - best-effort optional lint tool + warn(f"mermaid-cli install skipped (optional lint tool): {exc}") + return + if not which("mmdc"): + warn("Command 'mmdc' is still unavailable after installation.") + return + _ensure_browser()