diff --git a/docs/contributing/design/role/services/base.md b/docs/contributing/design/role/services/base.md index a591b8b353..635fa83f3d 100644 --- a/docs/contributing/design/role/services/base.md +++ b/docs/contributing/design/role/services/base.md @@ -57,6 +57,23 @@ Rules: - `canonical:` is only used on alias entries that resolve back to the primary service key. - `frontend` vs. `backend` is derived from the role name prefix, not stored in config. +## Dashboard Card Target ๐Ÿƒ + +A consumer's `dashboard` binding MAY name `card: ` to point its dashboard card, its Apps menu entry and its tile-reachability target at a domain other than the primary one: + +```yaml +dashboard: + bond: 1 + enabled: "{{ 'web-app-dashboard' in group_names }}" + shared: "{{ 'web-app-dashboard' in group_names }}" + card: element +``` + +Rules: + +- `card:` MUST be a key of the role's `canonical` mapping in `meta/domains.yml`; the card URL is `lookup('canonical_url', , )`. +- Without `card:` the card targets the primary domain via `lookup('tls', , 'url.base')`. + ## Service Discovery ๐Ÿ” Service discovery is built from role configs, not from a central registry file. diff --git a/group_vars/all/00_general.yml b/group_vars/all/00_general.yml index c85d227691..dcfdaa0467 100644 --- a/group_vars/all/00_general.yml +++ b/group_vars/all/00_general.yml @@ -83,8 +83,8 @@ CAPTCHA: KEY: "" SECRET: "" -RECAPTCHA_ENABLED: "{{ (CAPTCHA.RECAPTCHA.KEY | length > 0) and (CAPTCHA.RECAPTCHA.SECRET | length > 0) }}" +RECAPTCHA_ENABLED: "{{ (CAPTCHA.RECAPTCHA is defined) and (CAPTCHA.RECAPTCHA.KEY | length > 0) and (CAPTCHA.RECAPTCHA.SECRET | length > 0) }}" -HCAPTCHA_ENABLED: "{{ (CAPTCHA.HCAPTCHA.KEY | length > 0) and (CAPTCHA.HCAPTCHA.SECRET | length > 0) }}" +HCAPTCHA_ENABLED: "{{ (CAPTCHA.HCAPTCHA is defined) and (CAPTCHA.HCAPTCHA.KEY | length > 0) and (CAPTCHA.HCAPTCHA.SECRET | length > 0) }}" WORKSTATION_USER: "administrator" diff --git a/library/directory_batch.py b/library/directory_batch.py index 115026cabd..ea333ff8eb 100644 --- a/library/directory_batch.py +++ b/library/directory_batch.py @@ -19,18 +19,8 @@ Returns: diff: one ``before``/``after`` record per changed path, in diff mode. """ -import os -from pathlib import Path - from ansible.module_utils.basic import AnsibleModule - -ATTRIBUTES = ("mode", "owner", "group") - -ATTRIBUTE_SPEC = { - "mode": {"type": "raw"}, - "owner": {"type": "str"}, - "group": {"type": "str"}, -} +from ansible.module_utils.directory_batch import ATTRIBUTE_SPEC, ATTRIBUTES, ensure_all ARGUMENT_SPEC = { "entries": { @@ -47,10 +37,6 @@ MUTUALLY_EXCLUSIVE = [["entries", name] for name in ("paths", *ATTRIBUTES)] REQUIRED_ONE_OF = [["entries", "paths"]] -class DirectoryError(Exception): - pass - - def requested_entries(params): if params["entries"] is not None: return params["entries"] @@ -60,64 +46,6 @@ def requested_entries(params): ] -def path_state(path): - target = Path(path) - if target.is_symlink(): - return "link" - if target.is_dir(): - return "directory" - if target.exists(): - return "file" - return "absent" - - -def apply_attributes(module, path, entry, changed, diff): - file_args = module.load_file_common_arguments( - {"path": path, **{name: entry[name] for name in ATTRIBUTES}} - ) - return module.set_fs_attributes_if_different(file_args, changed, diff, expand=False) - - -def create(module, path, entry, diff): - changed = False - current = "" - for part in path.strip("/").split("/"): - current = f"{current}/{part}" - candidate = Path(current) - if candidate.exists(): - continue - try: - candidate.mkdir() - changed = True - except FileExistsError: - if not candidate.is_dir(): - raise - changed = apply_attributes(module, current, entry, changed, diff) - return changed - - -def ensure(module, entry): - path = entry["path"] - if not Path(path).is_absolute(): - raise DirectoryError(f"{path!r} is not an absolute path") - if Path(path).is_symlink(): - path = os.path.realpath(path) - state = path_state(path) - diff = {"before": {"path": path}, "after": {"path": path}} - if state == "directory": - return apply_attributes(module, path, entry, False, diff), diff - if state != "absent": - raise DirectoryError(f"{path} already exists as a {state}") - diff["before"]["state"] = state - diff["after"]["state"] = "directory" - if module.check_mode: - return True, diff - try: - return create(module, path, entry, diff), diff - except OSError as error: - raise DirectoryError(f"Could not create {path}: {error}") from error - - def main(): module = AnsibleModule( argument_spec=ARGUMENT_SPEC, @@ -125,21 +53,7 @@ def main(): required_one_of=REQUIRED_ONE_OF, supports_check_mode=True, ) - changed_paths = [] - diffs = [] - for entry in requested_entries(module.params): - try: - changed, diff = ensure(module, entry) - except DirectoryError as error: - module.fail_json( - msg=str(error), - path=entry["path"], - changed=bool(changed_paths), - changed_paths=changed_paths, - ) - if changed: - changed_paths.append(entry["path"]) - diffs.append(diff) + changed_paths, diffs = ensure_all(module, requested_entries(module.params)) result = {"changed": bool(changed_paths), "changed_paths": changed_paths} if module._diff: result["diff"] = diffs diff --git a/plugins/filter/baudolo_seed.py b/plugins/filter/baudolo_seed.py new file mode 100644 index 0000000000..2226116356 --- /dev/null +++ b/plugins/filter/baudolo_seed.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import base64 +import binascii +import csv +import io +import re + +COLUMNS = ("instance", "database", "username", "password") +CLUSTER_ROW = "*" +_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$") + + +def _serialize(rows) -> str: + buffer = io.StringIO() + writer = csv.writer(buffer, delimiter=";", lineterminator="\n") + writer.writerows(rows) + return buffer.getvalue() + + +def _database(value: str) -> str | None: + text = value.strip() + if text == CLUSTER_ROW: + return CLUSTER_ROW + if not text or text.lower() == "nan" or not _NAME_RE.match(text): + return None + return text + + +def baudolo_seeded(content: str) -> dict: + """The rows of a ``databases.csv`` and whether ``baudolo-seed`` wrote it. + + Args: + content: the file as base64, as ``slurp`` returns it; ``""`` for a file + that is absent or empty. + + Returns: + dict: + canonical - True only when the text is byte for byte what + ``baudolo-seed`` writes for these rows; undecodable or + hand-formatted text is False + rows - [instance, database, username, password] lists in file + order, empty when not canonical + """ + try: + text = base64.b64decode(content, validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return {"canonical": False, "rows": []} + if not text: + return {"canonical": True, "rows": []} + parsed = list(csv.reader(io.StringIO(text, newline=""), delimiter=";")) + if not parsed or tuple(parsed[0]) != COLUMNS: + return {"canonical": False, "rows": []} + rows = parsed[1:] + canonical = ( + all(len(row) == len(COLUMNS) for row in rows) and _serialize(parsed) == text + ) + return {"canonical": canonical, "rows": rows if canonical else []} + + +def baudolo_seed_current(seeded, instance, database, username, password) -> bool: + """Whether ``baudolo-seed`` with these arguments would leave the file as is. + + Args: + seeded: the state ``baudolo_seeded`` or ``baudolo_seed_apply`` returned. + instance, database, username, password: the ``baudolo-seed`` arguments. + + Returns: + True when the file is canonical and every row of this instance and + database already carries these credentials. + """ + name = _database(database) + if not seeded["canonical"] or name is None: + return False + matches = [row for row in seeded["rows"] if row[0] == instance and row[1] == name] + return bool(matches) and all( + row[2] == username and row[3] == password for row in matches + ) + + +def baudolo_seed_apply(seeded, instance, database, username, password) -> dict: + """The state after ``baudolo-seed`` ran with these arguments. + + Args: + seeded: the state ``baudolo_seeded`` or ``baudolo_seed_apply`` returned. + instance, database, username, password: the ``baudolo-seed`` arguments. + + Returns: + dict shaped like the ``baudolo_seeded`` result, with every row of this + instance and database carrying the credentials, or one row appended + when there was none; a non-canonical state stays non-canonical. + """ + name = _database(database) + if not seeded["canonical"] or name is None: + return {"canonical": False, "rows": []} + rows = [list(row) for row in seeded["rows"]] + matched = False + for row in rows: + if row[0] == instance and row[1] == name: + row[2:] = [username, password] + matched = True + if not matched: + rows.append([instance, name, username, password]) + return {"canonical": True, "rows": rows} + + +class FilterModule: + def filters(self): + return { + "baudolo_seeded": baudolo_seeded, + "baudolo_seed_current": baudolo_seed_current, + "baudolo_seed_apply": baudolo_seed_apply, + } diff --git a/plugins/lookup/roles_with_service.py b/plugins/lookup/roles_with_service.py index 5d8829b290..8d556fd8c3 100644 --- a/plugins/lookup/roles_with_service.py +++ b/plugins/lookup/roles_with_service.py @@ -11,6 +11,11 @@ domain, so both follow the onion substitution a Tor deployment applies. A consumer that compares it against what a page actually renders would otherwise never match on an onion host. +For the ``dashboard`` service a role MAY name ``services.dashboard.card: +``; its entry then carries ``domains.canonical.`` and +``lookup('canonical_url', , )`` instead of the primary domain, so +the tile target matches the card ``docker_cards`` renders. + A role keeps declaring the service for inventory completeness but can opt out of this consumer-target list by setting ``services..scrape: false`` or ``services..track: false``. @@ -60,6 +65,7 @@ from ansible.plugins.lookup import LookupBase from utils.domains.primary_domain import get_primary_domain from utils.networks.reachability import network, network_of +from utils.roles.applications.dashboard import DASHBOARD_SERVICE, card_target_key from utils.roles.applications.mcp import ( DEFAULT_MCP_TRANSPORT, derive_allowed_consumers, @@ -100,6 +106,30 @@ def _resolve_canonical_domain(role_id: str, app_config: dict[str, Any]) -> str: return get_primary_domain({role_id: domains["canonical"]}, role_id) +def _resolve_named_canonical_domain( + role_id: str, app_config: dict[str, Any], key: str +) -> str: + """Return the domain ``domains.canonical.`` of one role. + + Args: + role_id: the role whose domains are read. + app_config: the role's merged applications entry. + key: the named canonical, e.g. ``element``. + + Raises: + AnsibleError: the role declares no such named canonical. + """ + domains = app_config.get("domains") + canonical = domains.get("canonical") if isinstance(domains, dict) else None + domain = canonical.get(key) if isinstance(canonical, dict) else None + if not isinstance(domain, str) or not domain: + raise AnsibleError( + f"roles_with_service: '{role_id}' has no canonical '{key}' " + f"(available: {sorted(canonical) if isinstance(canonical, dict) else []})" + ) + return domain + + def _deployed_roles(scope: str, vars_: dict[str, Any]) -> set[str] | None: """Return the role ids the scope admits, or None for no restriction. @@ -192,6 +222,7 @@ class LookupModule(LookupBase): tls_lookup = lookup_loader.get( "tls", loader=self._loader, templar=self._templar ) + lists_dashboard_cards = not topic and service_name == DASHBOARD_SERVICE results: list[dict[str, Any]] = [] for role_id, app_config in applications.items(): @@ -222,12 +253,24 @@ class LookupModule(LookupBase): canonical = _resolve_canonical_domain(str(role_id), app_config) if not canonical and direction is None and not include_headless: continue + card_key = ( + card_target_key(str(role_id), block) if lists_dashboard_cards else "" + ) canonical_url = "" - if canonical: + if card_key: + canonical = _resolve_named_canonical_domain( + str(role_id), app_config, card_key + ) + resolved = lookup_loader.get( + "canonical_url", loader=self._loader, templar=self._templar + ).run([str(role_id), card_key], variables=variables) + canonical_url = str(resolved[0]).rstrip("/") + elif canonical: resolved = tls_lookup.run( [str(role_id), "url.base"], variables=variables ) canonical_url = str(resolved[0]).rstrip("/") + if canonical_url: canonical = urlsplit(canonical_url).hostname or canonical if handover_lookup is not None and not _renders_in( pass_network, str(role_id), canonical, handover_lookup, variables diff --git a/roles/svc-bkp-volume-2-local/README.md b/roles/svc-bkp-volume-2-local/README.md index 1703a09e03..9713112ba1 100644 --- a/roles/svc-bkp-volume-2-local/README.md +++ b/roles/svc-bkp-volume-2-local/README.md @@ -9,6 +9,7 @@ File payloads are captured with rsync hard-link snapshots; databases register th This role installs the `baudolo` CLI, lays out the on-host backup tree, deploys the systemd service that drives the periodic run, and wires the cleanup-of-failed-backups dependency so partial snapshots are not retained. Database seeding for individual apps is contributed by the consumer roles via `tasks/03_seed-database-to-backup.yml`, which they include conditionally once `svc-bkp-volume-2-local` is in `group_names`. +The first seed of a play reads `databases.csv` once through `tasks/05_read-seeded-databases.yml`; a consumer whose row already holds its credentials skips `baudolo-seed`, every other consumer still calls it at its own point in the play. ## Schema diff --git a/roles/svc-bkp-volume-2-local/tasks/02_reset.yml b/roles/svc-bkp-volume-2-local/tasks/02_reset.yml index 6b5a5fc7c0..e2853b525f 100644 --- a/roles/svc-bkp-volume-2-local/tasks/02_reset.yml +++ b/roles/svc-bkp-volume-2-local/tasks/02_reset.yml @@ -3,3 +3,7 @@ ansible.builtin.file: path: "{{ FILE_DATABASE_SECRETS }}" state: absent + +- name: "๐Ÿงฎ Forget the seeded database definitions along with the file" + ansible.builtin.set_fact: + svc_bkp_volume_2_local_seeded: "{{ '' | baudolo_seeded }}" diff --git a/roles/svc-bkp-volume-2-local/tasks/03_seed-database-to-backup.yml b/roles/svc-bkp-volume-2-local/tasks/03_seed-database-to-backup.yml index f7da52467e..7a665afad5 100644 --- a/roles/svc-bkp-volume-2-local/tasks/03_seed-database-to-backup.yml +++ b/roles/svc-bkp-volume-2-local/tasks/03_seed-database-to-backup.yml @@ -1,6 +1,7 @@ --- -- include_tasks: 01_install.yml - when: run_once_svc_bkp_volume_2_local_install is not defined +- name: "๐Ÿ“– Read the seeded database definitions once per host" + include_tasks: 05_read-seeded-databases.yml + when: svc_bkp_volume_2_local_seeded is not defined - name: "๐Ÿ› BKP | Debug seed context (sanitized)" ansible.builtin.debug: @@ -47,9 +48,15 @@ Note: database_name may be an empty string to represent "all databases". changed_when: false -- name: ๐ŸŒฑ Seed database definition via baudolo-seed (system binary) - ansible.builtin.command: - argv: +- name: "๐Ÿงฎ Compare the database definition with the seeded one" + ansible.builtin.set_fact: + svc_bkp_volume_2_local_seed_argv: "{{ svc_bkp_volume_2_local_seed_command }}" + svc_bkp_volume_2_local_seed_current: >- + {{ svc_bkp_volume_2_local_seeded | baudolo_seed_current(*svc_bkp_volume_2_local_seed_command[2:]) }} + svc_bkp_volume_2_local_seeded: >- + {{ svc_bkp_volume_2_local_seeded | baudolo_seed_apply(*svc_bkp_volume_2_local_seed_command[2:]) }} + vars: + svc_bkp_volume_2_local_seed_command: - baudolo-seed - "{{ FILE_DATABASE_SECRETS | string }}" - "{{ database_instance | string }}" @@ -57,5 +64,11 @@ - "{{ database_username | string }}" - "{{ database_password | string }}" no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: ๐ŸŒฑ Seed database definition via baudolo-seed (system binary) + ansible.builtin.command: + argv: "{{ svc_bkp_volume_2_local_seed_argv }}" + when: not (svc_bkp_volume_2_local_seed_current | bool) + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" # todo: add flock to baudolo-seed upstream, then async seeding becomes safe again. throttle: 1 diff --git a/roles/svc-bkp-volume-2-local/tasks/05_read-seeded-databases.yml b/roles/svc-bkp-volume-2-local/tasks/05_read-seeded-databases.yml new file mode 100644 index 0000000000..5be6655631 --- /dev/null +++ b/roles/svc-bkp-volume-2-local/tasks/05_read-seeded-databases.yml @@ -0,0 +1,22 @@ +--- +- include_tasks: 01_install.yml + when: run_once_svc_bkp_volume_2_local_install is not defined + +- name: "๐Ÿ”Ž Check whether {{ FILE_DATABASE_SECRETS }} exists" + ansible.builtin.stat: + path: "{{ FILE_DATABASE_SECRETS }}" + register: svc_bkp_volume_2_local_seeded_file + +- name: "๐Ÿ“– Read the database definitions seeded on this host" + ansible.builtin.slurp: + src: "{{ FILE_DATABASE_SECRETS }}" + register: svc_bkp_volume_2_local_seeded_raw + when: svc_bkp_volume_2_local_seeded_file.stat.exists + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "๐Ÿงฎ Remember the seeded database definitions for this play" + ansible.builtin.set_fact: + svc_bkp_volume_2_local_seeded: >- + {{ (svc_bkp_volume_2_local_seeded_raw.content if svc_bkp_volume_2_local_seeded_file.stat.exists else '') + | baudolo_seeded }} + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" diff --git a/roles/sys-ctl-hlth-csp/defaults/main.yml b/roles/sys-ctl-hlth-csp/defaults/main.yml index 1e60bfeba0..6d4f993411 100644 --- a/roles/sys-ctl-hlth-csp/defaults/main.yml +++ b/roles/sys-ctl-hlth-csp/defaults/main.yml @@ -19,3 +19,6 @@ HEALTH_CSP_CLEARNET_TIMEOUT_MS: 45000 # Seconds added per vhost on top of its navigation budget when sizing TimeoutStartSec HEALTH_CSP_DOMAIN_RESERVE_SECONDS: 5 + +# Number of vhosts the checker probes in parallel, each in its own browser context +HEALTH_CSP_CONCURRENCY: 4 diff --git a/roles/sys-ctl-hlth-csp/files/python/script.py b/roles/sys-ctl-hlth-csp/files/python/script.py index 0624c662e9..3038860899 100755 --- a/roles/sys-ctl-hlth-csp/files/python/script.py +++ b/roles/sys-ctl-hlth-csp/files/python/script.py @@ -206,6 +206,7 @@ def build_docker_cmd( proxy_bypass: str = "", confine_to: str = "", confine_allow: list[str] | None = None, + concurrency: int = 0, ) -> list[str]: cmd = ["container", "run", "--rm"] @@ -233,6 +234,9 @@ def build_docker_cmd( if timeout_ms: cmd.extend(["--timeout", str(timeout_ms)]) + if concurrency: + cmd.extend(["--concurrency", str(concurrency)]) + if ignore_network_blocks_from: cmd.append("--ignore-network-blocks-from") cmd.extend(ignore_network_blocks_from) @@ -248,40 +252,18 @@ def build_docker_cmd( return cmd -def run_checker( - image: str, - urls: list[str], - short_mode: bool, - ignore_network_blocks_from: list[str], - always_pull: bool, - use_host_network: bool = True, - proxy: str = "", - timeout_ms: int = 0, - accept_status: list[str] | None = None, - proxy_bypass: str = "", - confine_to: str = "", - confine_allow: list[str] | None = None, -) -> int: - """ - Runs the CSP checker container and returns its exit code. - Always uses run. +def run_checker(image: str, always_pull: bool, **cmd_options: object) -> int: + """Run the CSP checker container and return its exit code. + + Args: + image: the csp-checker image. + always_pull: pull the image before the run. + cmd_options: every further keyword argument of build_docker_cmd. """ if always_pull: subprocess.run(["container", "pull", image], check=False) - cmd = build_docker_cmd( - image=image, - urls=urls, - short_mode=short_mode, - ignore_network_blocks_from=ignore_network_blocks_from, - use_host_network=use_host_network, - proxy=proxy, - timeout_ms=timeout_ms, - accept_status=accept_status, - proxy_bypass=proxy_bypass, - confine_to=confine_to, - confine_allow=confine_allow, - ) + cmd = build_docker_cmd(image=image, **cmd_options) try: result = subprocess.run(cmd, check=False) @@ -377,6 +359,15 @@ def main() -> None: "own default. 0 leaves that default in place." ), ) + parser.add_argument( + "--concurrency", + type=int, + default=0, + help=( + "Number of URLs the checker probes in parallel, each in its own " + "browser context. 0 leaves the checker's own default in place." + ), + ) parser.add_argument( "--tor-proxy", default="", @@ -485,6 +476,7 @@ def main() -> None: always_pull=bool(args.always_pull), use_host_network=not bool(args.no_host_network), accept_status=accept_status, + concurrency=args.concurrency, **batch, ) rc = rc or batch_rc diff --git a/roles/sys-ctl-hlth-csp/meta/services.yml b/roles/sys-ctl-hlth-csp/meta/services.yml index a0065de396..e70d0b5d3a 100644 --- a/roles/sys-ctl-hlth-csp/meta/services.yml +++ b/roles/sys-ctl-hlth-csp/meta/services.yml @@ -9,4 +9,4 @@ csp: # nocheck: image-healthcheck crawler run as `container run --rm` from a systemd timer, it exits; the role renders no compose service to carry a probe csp-checker: image: ghcr.io/kevinveenbirkenbach/csp-checker - version: 3.1.3 + version: 3.2.0 diff --git a/roles/sys-ctl-hlth-csp/tasks/00_core.yml b/roles/sys-ctl-hlth-csp/tasks/00_core.yml index 4fc2534006..15f7b83e3e 100644 --- a/roles/sys-ctl-hlth-csp/tasks/00_core.yml +++ b/roles/sys-ctl-hlth-csp/tasks/00_core.yml @@ -48,6 +48,7 @@ '--image=' ~ HEALTH_CSP_IMAGE, '--short', '--clearnet-timeout=' ~ HEALTH_CSP_CLEARNET_TIMEOUT_MS, + '--concurrency=' ~ HEALTH_CSP_CONCURRENCY, (HEALTH_CSP_ALWAYS_PULL | bool) | ternary('--always-pull', ''), (HEALTH_CSP_IGNORE_NETWORK_BLOCKS_FROM | length > 0) | ternary( '--ignore-network-blocks-from ' ~ (HEALTH_CSP_IGNORE_NETWORK_BLOCKS_FROM | join(' ')), diff --git a/roles/sys-front-tls-letsencrypt/tasks/issue.yml b/roles/sys-front-tls-letsencrypt/tasks/issue.yml index 6203f7d4d8..508360f603 100644 --- a/roles/sys-front-tls-letsencrypt/tasks/issue.yml +++ b/roles/sys-front-tls-letsencrypt/tasks/issue.yml @@ -9,20 +9,12 @@ vars: domain: "{{ _le_cert_domain }}" -- name: "๐Ÿ”Ž TLS | (letsencrypt) re-check cert files exist" - ansible.builtin.stat: - path: "{{ item }}" - register: _le_stat2 - loop: - - "{{ _cert_file }}" - - "{{ _cert_key }}" - changed_when: false +- name: "๐Ÿ”Ž TLS | (letsencrypt) refresh the certificate inventory" + include_tasks: utils/certificates.yml - name: "๐Ÿ›‘ TLS | (letsencrypt) fail if still missing" ansible.builtin.fail: msg: >- Let's Encrypt certificate files not found after issuance: cert={{ _cert_file }} key={{ _cert_key }} - when: > - (not _le_stat2.results[0].stat.exists | default(false)) or - (not _le_stat2.results[1].stat.exists | default(false)) + when: not ([_cert_file, _cert_key] is subset(tls_letsencrypt_certificates)) diff --git a/roles/sys-front-tls-letsencrypt/tasks/main.yml b/roles/sys-front-tls-letsencrypt/tasks/main.yml index d16545ecfe..cec149c4c1 100644 --- a/roles/sys-front-tls-letsencrypt/tasks/main.yml +++ b/roles/sys-front-tls-letsencrypt/tasks/main.yml @@ -20,17 +20,10 @@ but resolver returned enabled={{ _tls_enabled }}, mode={{ _tls_mode }}. changed_when: false -- name: "๐Ÿ”Ž TLS | (letsencrypt) check cert files exist" - ansible.builtin.stat: - path: "{{ item }}" - register: _le_stat - loop: - - "{{ _cert_file }}" - - "{{ _cert_key }}" - changed_when: false +- name: "๐Ÿ”Ž TLS | (letsencrypt) take the certificate inventory once per play" + include_tasks: utils/certificates.yml + when: tls_letsencrypt_certificates is not defined - name: "๐Ÿ“œ TLS | (letsencrypt) issue cert if missing" include_tasks: issue.yml - when: > - (not _le_stat.results[0].stat.exists | default(false)) or - (not _le_stat.results[1].stat.exists | default(false)) + when: not ([_cert_file, _cert_key] is subset(tls_letsencrypt_certificates)) diff --git a/roles/sys-front-tls-letsencrypt/tasks/utils/certificates.yml b/roles/sys-front-tls-letsencrypt/tasks/utils/certificates.yml new file mode 100644 index 0000000000..e3aea6f4d6 --- /dev/null +++ b/roles/sys-front-tls-letsencrypt/tasks/utils/certificates.yml @@ -0,0 +1,22 @@ +--- +- name: "๐Ÿ”Ž TLS | (letsencrypt) check the live directory" + ansible.builtin.stat: + path: "{{ LETSENCRYPT_LIVE_PATH }}" + register: tls_letsencrypt_live + +- name: "๐Ÿ”Ž TLS | (letsencrypt) find every certificate file in one call" + ansible.builtin.find: + paths: "{{ LETSENCRYPT_LIVE_PATH }}" + patterns: "{{ [_cert_file, _cert_key] | map('basename') | unique }}" + file_type: any + follow: true + recurse: true + depth: 2 + register: tls_letsencrypt_found + when: tls_letsencrypt_live.stat.exists + +- name: "๐Ÿ“Œ TLS | (letsencrypt) remember the certificate files" + ansible.builtin.set_fact: + tls_letsencrypt_certificates: >- + {{ tls_letsencrypt_found.files | map(attribute='path') | list + if tls_letsencrypt_live.stat.exists else [] }} diff --git a/roles/sys-front-tls/tasks/01_http.yml b/roles/sys-front-tls/tasks/01_http.yml deleted file mode 100644 index a61da5fdbd..0000000000 --- a/roles/sys-front-tls/tasks/01_http.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: ๐Ÿ“‚ Include dependency 'sys-ctl-mtn-cert-renew' - include_role: - name: sys-ctl-mtn-cert-renew - when: - - run_once_sys_ctl_mtn_cert_renew is not defined - - lookup('tls', domain, 'mode') == 'letsencrypt' - -- name: ๐Ÿ“ Create HTTP->HTTPS handler per domain - stack_host_template: - src: "http.conf.j2" - dest: "{{ lookup('nginx', 'files.domain', domain, protocol='http') }}" - notify: reload-openresty diff --git a/roles/sys-front-tls/tasks/main.yml b/roles/sys-front-tls/tasks/main.yml index bc741aecd3..7c13794efa 100644 --- a/roles/sys-front-tls/tasks/main.yml +++ b/roles/sys-front-tls/tasks/main.yml @@ -14,6 +14,10 @@ name: sys-front-tls-selfsigned when: tls_mode_effective == 'self_signed' -- name: "๐Ÿ“‚ Load http redirect routines" - include_tasks: 01_http.yml - when: lookup('tls', domain, 'enabled') | bool +- name: ๐Ÿ“‚ Include dependency 'sys-ctl-mtn-cert-renew' + include_role: + name: sys-ctl-mtn-cert-renew + when: + - run_once_sys_ctl_mtn_cert_renew is not defined + - lookup('tls', domain, 'enabled') | bool + - tls_mode_effective == 'letsencrypt' diff --git a/roles/sys-stk-front-proxy/tasks/vhost.yml b/roles/sys-stk-front-proxy/tasks/vhost.yml index 1e4151ca76..320fb1f3b9 100644 --- a/roles/sys-stk-front-proxy/tasks/vhost.yml +++ b/roles/sys-stk-front-proxy/tasks/vhost.yml @@ -44,9 +44,15 @@ webserver_vhost_flavour: basic - name: "๐Ÿš€ Deploy NGINX config for '{{ front_proxy_domain }}'" - stack_host_template: - src: "{{ front_proxy_template_src }}" - dest: "{{ front_proxy_domain_conf_dst_resolved }}" + stack_host_contents: + files: >- + {{ ([{'path': lookup('nginx', 'files.domain', front_proxy_domain, protocol='http'), + 'content': lookup('template', lookup('path_absolute', 'roles/sys-front-tls/templates/http.conf.j2'), + template_vars={'domain': front_proxy_domain})}] + if lookup('tls', front_proxy_domain, 'enabled') | bool else []) + + [{'path': front_proxy_domain_conf_dst_resolved, + 'content': lookup('template', front_proxy_template_src)}] }} + mode: "0644" vars: application_id: "{{ front_proxy_application_id }}" domain: "{{ front_proxy_domain }}" @@ -63,7 +69,7 @@ ansible.builtin.meta: flush_handlers - name: "๐Ÿ” Re-check '{{ front_proxy_domain }}' after a config change" - when: nginx_conf.changed + when: front_proxy_vhost_changed block: - name: "๐Ÿ” Verify '{{ front_proxy_domain }}' is served after the reload" uri_probe: diff --git a/roles/sys-stk-front-proxy/vars/main.yml b/roles/sys-stk-front-proxy/vars/main.yml index 840960dd45..85a105bd42 100644 --- a/roles/sys-stk-front-proxy/vars/main.yml +++ b/roles/sys-stk-front-proxy/vars/main.yml @@ -1,5 +1,6 @@ --- front_proxy_probe_status_codes: "{{ front_proxy_status_codes + front_proxy_upstream_down_status_codes }}" +front_proxy_vhost_changed: "{{ nginx_conf.changed and front_proxy_domain_conf_dst_resolved in nginx_conf.changed_paths }}" front_proxy_deferred_urls: "{{ dict(front_proxy_deferred_probes.keys() | zip(front_proxy_deferred_probes.values() | map(attribute='url'))) }}" front_proxy_deferred_answers: >- {{ diff --git a/roles/sys-svc-compose/library/compose_instance_prepare.py b/roles/sys-svc-compose/library/compose_instance_prepare.py new file mode 100644 index 0000000000..feb0996a5a --- /dev/null +++ b/roles/sys-svc-compose/library/compose_instance_prepare.py @@ -0,0 +1,65 @@ +"""Ensure a compose instance's directories and probe its state in one call. + +Args: + directories: paths ensured like ``directory_batch`` ``paths``. + mode: mode of every path in ``directories``. + instance: compose project directory ``compose ps`` runs in. + pending_marker: path reported in ``pending``. + running_check: whether to run ``compose ps -q --filter status=running`` + in ``instance``. + +Returns: + changed_paths: the directories that were created or whose mode changed. + diff: one ``before``/``after`` record per changed directory, in diff mode. + pending: whether ``pending_marker`` exists, a symlink not followed. + running: ``rc``, ``stdout`` and ``stderr`` of ``compose ps``, each stream + stripped of trailing newlines; null when ``running_check`` is false or + in check mode. +""" + +import os + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.directory_batch import ensure_all + +ARGUMENT_SPEC = { + "directories": {"type": "list", "elements": "path", "required": True}, + "mode": {"type": "raw", "required": True}, + "instance": {"type": "path", "required": True}, + "pending_marker": {"type": "path", "required": True}, + "running_check": {"type": "bool", "required": True}, +} + +RUNNING_ARGS = ("ps", "-q", "--filter", "status=running") + + +def running(module): + if not module.params["running_check"] or module.check_mode: + return None + compose = module.get_bin_path("compose", required=True) + rc, stdout, stderr = module.run_command( + [compose, *RUNNING_ARGS], cwd=module.params["instance"] + ) + return {"rc": rc, "stdout": stdout.rstrip("\r\n"), "stderr": stderr.rstrip("\r\n")} + + +def main(): + module = AnsibleModule(argument_spec=ARGUMENT_SPEC, supports_check_mode=True) + entries = [ + {"path": path, "mode": module.params["mode"], "owner": None, "group": None} + for path in module.params["directories"] + ] + changed_paths, diffs = ensure_all(module, entries) + result = { + "changed": bool(changed_paths), + "changed_paths": changed_paths, + "pending": os.path.lexists(module.params["pending_marker"]), + "running": running(module), + } + if module._diff: + result["diff"] = diffs + module.exit_json(**result) + + +if __name__ == "__main__": + main() diff --git a/roles/sys-svc-compose/tasks/02_stack_host_prep.yml b/roles/sys-svc-compose/tasks/02_stack_host_prep.yml index 581abcd26a..21f3a8f598 100644 --- a/roles/sys-svc-compose/tasks/02_stack_host_prep.yml +++ b/roles/sys-svc-compose/tasks/02_stack_host_prep.yml @@ -1,8 +1,12 @@ --- -- name: "๐Ÿ“ Create sys-svc-compose dirs (with parents) for '{{ application_id }}'" - directory_batch: - paths: "{{ lookup('container', application_id, 'directories').values() | list }}" +- name: "๐Ÿ“ Create sys-svc-compose dirs and probe '{{ application_id }}'" + compose_instance_prepare: + directories: "{{ lookup('container', application_id, 'directories').values() | list }}" mode: "0755" + instance: "{{ lookup('container', application_id, 'directories.instance') }}" + pending_marker: "{{ COMPOSE_UP_PENDING_MARKER }}" + running_check: "{{ compose_mode == 'compose' }}" + register: compose_instance_before_files - name: "๐Ÿ›ก๏ธ Ensure PHP hardening assets are installed" ansible.builtin.include_role: diff --git a/roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml b/roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml index 249d1a6b46..50450b7302 100644 --- a/roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml +++ b/roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml @@ -43,21 +43,18 @@ - name: "๐Ÿ” resolve_host_cid | resolve local container id for '{{ resolve_service_key }}'" ansible.builtin.shell: | - {% if IS_SWARM_MODE %} container ps \ --filter "label=com.docker.swarm.service.name={{ _resolve_service_name }}" \ --filter status=running {{ resolve_cid_filters | default('') }} \ --format '{{ "{{.ID}}" }}' | head -1 - {% else %} - echo {{ resolve_compose_cid | quote }} - {% endif %} register: _resolve_cid changed_when: false retries: 30 delay: 2 - until: IS_COMPOSE_MODE or (_resolve_cid.stdout | trim) | length > 0 + until: (_resolve_cid.stdout | trim) | length > 0 delegate_to: "{{ lookup('vars', resolve_output_prefix ~ '_HOST_NODE') }}" + when: IS_SWARM_MODE - name: "๐Ÿ”ง resolve_host_cid | set {{ resolve_output_prefix }}_LOCAL_CID" ansible.builtin.set_fact: - "{{ resolve_output_prefix }}_LOCAL_CID": "{{ _resolve_cid.stdout | trim }}" + "{{ resolve_output_prefix }}_LOCAL_CID": "{{ (_resolve_cid.stdout | trim) if IS_SWARM_MODE else (resolve_compose_cid | trim) }}" diff --git a/roles/sys-svc-compose/tasks/utils/up.yml b/roles/sys-svc-compose/tasks/utils/up.yml index f0d211b389..7d635ca018 100644 --- a/roles/sys-svc-compose/tasks/utils/up.yml +++ b/roles/sys-svc-compose/tasks/utils/up.yml @@ -1,46 +1,29 @@ --- - name: "๐Ÿ” Check running containers for '{{ application_id }}'" - command: >- - compose ps -q --filter status=running - args: - chdir: "{{ lookup('container', application_id, 'directories.instance') }}" - register: docker_ps - changed_when: > - (docker_ps.stdout | trim) == "" - failed_when: > - docker_ps.rc != 0 - and ( - (docker_ps.stderr | default('')) - | regex_search('(no configuration file provided|no such file or directory|env file .* not found)') is none - ) + ansible.builtin.assert: + that: >- + compose_instance_before_files.running.rc == 0 + or compose_instance_before_files.running.stderr + is search('(no configuration file provided|no such file or directory|env file .* not found)') + fail_msg: "compose ps exited {{ compose_instance_before_files.running.rc }}: {{ compose_instance_before_files.running.stderr }}" + quiet: true + changed_when: (compose_instance_before_files.running.stdout | trim) == "" when: - - > - not ( - docker_compose_template.changed | default(false) - or - env_template.changed | default(false) - ) - - compose_mode == 'compose' + - compose_instance_before_files.running is not none + - not (docker_compose_template.changed or env_template.changed or compose_override_template.changed) notify: compose-up -- name: "๐Ÿ” Resume a compose up that a failed run left pending" - when: compose_mode == 'compose' - block: - - name: "๐Ÿ” Check for a compose up a failed run left pending" - ansible.builtin.stat: - path: "{{ COMPOSE_UP_PENDING_MARKER }}" - register: compose_up_pending - - - name: "๐Ÿ”” Resume the compose up a failed run left pending" - ansible.builtin.debug: - msg: "{{ COMPOSE_UP_PENDING_MARKER }} is still present, so the last compose up never completed" - changed_when: true - notify: - - compose-build - - compose-up - when: - - compose_up_pending.stat.exists - - compose_up_marker is not changed +- name: "๐Ÿ”” Resume the compose up a failed run left pending" + ansible.builtin.debug: + msg: "{{ COMPOSE_UP_PENDING_MARKER }} is still present, so the last compose up never completed" + changed_when: true + notify: + - compose-build + - compose-up + when: + - compose_mode == 'compose' + - compose_instance_before_files.pending + - compose_up_marker is not changed - name: "๐Ÿ”” Notify swarm deploy (under DEPLOYMENT_MODE == swarm)" ansible.builtin.debug: diff --git a/roles/web-app-dashboard/lookup_plugins/docker_cards.py b/roles/web-app-dashboard/lookup_plugins/docker_cards.py index ff6a52294d..5d08d82b0c 100644 --- a/roles/web-app-dashboard/lookup_plugins/docker_cards.py +++ b/roles/web-app-dashboard/lookup_plugins/docker_cards.py @@ -9,6 +9,7 @@ from ansible.plugins.lookup import LookupBase from utils.cache.files import read_text from utils.cache.yaml import load_yaml_any from utils.roles.applications.config import get +from utils.roles.applications.dashboard import DASHBOARD_SERVICE, card_target_key from utils.roles.mapping import ( ROLE_FILE_META_INFO, ROLE_FILE_META_MAIN, @@ -28,7 +29,8 @@ class LookupModule(LookupBase): - Retrieves the icon class from meta/info.yml.logo.class - Retrieves the display flag from meta/info.yml.display - Retrieves the tags from galaxy_info.galaxy_tags - - Builds the URL using the 'domains' variable + - Builds the URL from the primary domain, or from the domains.canonical + key named by services.dashboard.card - Sets the iframe flag from applications Only cards whose application_id is included in the variable group_names are returned. @@ -132,28 +134,32 @@ class LookupModule(LookupBase): self._templar.template(domain_url).strip() if domain_url else "" ) - url = "" - if domain_url: - try: - tls_lookup = lookup_loader.get( - "tls", loader=self._loader, templar=self._templar - ) - base_url = tls_lookup.run( - [application_id, "url.base"], variables=variables - )[0] - url = str(base_url).strip().rstrip("/") - except Exception as e: - raise AnsibleError( - f"Error building URL via tls for '{application_id}': {e}" - ) from e - dashboard_cfg = get( applications, application_id, - "services.dashboard", + f"services.{DASHBOARD_SERVICE}", strict=False, default={}, ) + + url = "" + if domain_url: + card_key = card_target_key(application_id, dashboard_cfg) + try: + if card_key: + target_url = lookup_loader.get( + "canonical_url", loader=self._loader, templar=self._templar + ).run([application_id, card_key], variables=variables)[0] + else: + target_url = lookup_loader.get( + "tls", loader=self._loader, templar=self._templar + ).run([application_id, "url.base"], variables=variables)[0] + url = str(target_url).strip().rstrip("/") + except Exception as e: + raise AnsibleError( + f"Error building the card URL for '{application_id}': {e}" + ) from e + if isinstance(dashboard_cfg, dict) and "iframe" in dashboard_cfg: iframe = dashboard_cfg["iframe"] else: diff --git a/roles/web-app-discourse/tasks/03_launcher.yml b/roles/web-app-discourse/tasks/03_launcher.yml index 69e8f9b316..ceeb24ed5d 100644 --- a/roles/web-app-discourse/tasks/03_launcher.yml +++ b/roles/web-app-discourse/tasks/03_launcher.yml @@ -3,6 +3,13 @@ ansible.builtin.set_fact: DISCOURSE_REBUILT: false +- name: "๐Ÿงน Retire the launcher checkout of the URL-named layout" + include_tasks: + file: launcher/legacy_checkout.yml + apply: + become: true + when: (DISCOURSE_REPOSITORY_LEGACY_RELPATH | split('/') | first) not in ['..', DISCOURSE_REPOSITORY_DIR | basename] + - name: โฌ‡๏ธ Pull Discourse docker repository git: repo: "{{ DISCOURSE_REPOSITORY_URL }}" diff --git a/roles/web-app-discourse/tasks/launcher/legacy_checkout.yml b/roles/web-app-discourse/tasks/launcher/legacy_checkout.yml new file mode 100644 index 0000000000..a0820a7f93 --- /dev/null +++ b/roles/web-app-discourse/tasks/launcher/legacy_checkout.yml @@ -0,0 +1,16 @@ +--- +- name: "๐Ÿ—‘๏ธ Remove the URL-named launcher checkout" + ansible.builtin.file: + path: "{{ [ DISCOURSE_SERVICES_DIR, DISCOURSE_REPOSITORY_LEGACY_RELPATH ] | path_join }}" + state: absent + +- name: "๐Ÿ—‘๏ธ Prune the emptied parents of the URL-named checkout" + ansible.builtin.command: + argv: + - rmdir + - --ignore-fail-on-non-empty + - --parents + - "{{ DISCOURSE_REPOSITORY_LEGACY_RELPATH | dirname }}" + chdir: "{{ DISCOURSE_SERVICES_DIR }}" + removes: "{{ [ DISCOURSE_SERVICES_DIR, DISCOURSE_REPOSITORY_LEGACY_RELPATH | dirname ] | path_join }}" + when: DISCOURSE_REPOSITORY_LEGACY_RELPATH is search('/') diff --git a/roles/web-app-discourse/vars/main.yml b/roles/web-app-discourse/vars/main.yml index 9fad8a3689..454daf4465 100644 --- a/roles/web-app-discourse/vars/main.yml +++ b/roles/web-app-discourse/vars/main.yml @@ -24,7 +24,9 @@ DISCOURSE_DATABASE_TYPE: "{{ DISCOURSE_DATABASE.type }}" DISCOURSE_DATABASE_CONTAINER: "{{ DISCOURSE_DATABASE.container }}" # nocheck: container-var-name-lookup references the shared database container, not this role's service DISCOURSE_DATABASE_IS_SHARED: "{{ DISCOURSE_DATABASE.shared }}" -DISCOURSE_REPOSITORY_DIR: "{{ lookup('container', application_id, 'directories.services') }}{{ lookup('config', application_id, 'services.discourse.repository') }}/" +DISCOURSE_SERVICES_DIR: "{{ lookup('container', application_id, 'directories.services') }}" +DISCOURSE_REPOSITORY_DIR: "{{ [ DISCOURSE_SERVICES_DIR, 'discourse_docker' ] | path_join }}" +DISCOURSE_REPOSITORY_LEGACY_RELPATH: "{{ (DISCOURSE_SERVICES_DIR ~ DISCOURSE_REPOSITORY_URL) | relpath(DISCOURSE_SERVICES_DIR) }}" DISCOURSE_CONTAINERS_DIR: "{{ [ DISCOURSE_REPOSITORY_DIR, 'containers' ] | path_join }}" DISCOURSE_APPLICATION_YML_DEST: "{{ [ DISCOURSE_CONTAINERS_DIR, DISCOURSE_CONTAINER ~ '.yml' ] | path_join }}" diff --git a/roles/web-app-matrix/meta/services.yml b/roles/web-app-matrix/meta/services.yml index efa1da5705..3433f811cd 100644 --- a/roles/web-app-matrix/meta/services.yml +++ b/roles/web-app-matrix/meta/services.yml @@ -22,6 +22,7 @@ dashboard: bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" + card: element matomo: bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" diff --git a/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 b/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 index 075dfc8c4b..c1bf62cec7 100644 --- a/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 +++ b/roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2 @@ -155,6 +155,8 @@ matrix_client_element_branding_auth_header_logo_url: {{ MATRIX_DESIGN_LOGO_URL | matrix_client_element_branding_welcome_background_url: "" matrix_client_element_configuration_extension_json: '{"default_theme": null}' {% endif %} +matrix_synapse_configuration_extension_yaml: | + web_client_location: "{{ lookup('canonical_url', application_id, 'element') }}" matrix_bridge_appservice_irc_enabled: {{ MATRIX_BRIDGE.appservice_irc | string | lower }} matrix_bridge_heisenbridge_enabled: {{ MATRIX_BRIDGE.heisenbridge | string | lower }} diff --git a/roles/web-app-nextcloud/README.md b/roles/web-app-nextcloud/README.md index 8dbf608add..dfe124a210 100644 --- a/roles/web-app-nextcloud/README.md +++ b/roles/web-app-nextcloud/README.md @@ -64,6 +64,19 @@ After the compose handlers ran, [`tasks/02_manager_ops/03_upgrade.yml`](./tasks/ The steps run in compose mode only. A swarm deploy that finds an older major stops with the planned steps; pin one major above the installed one per deploy there. +### Post-Upgrade Pass + +[`tasks/02_manager_ops/03_upgrade.yml`](./tasks/02_manager_ops/03_upgrade.yml) includes [`03_upgrade/01_post_upgrade.yml`](./tasks/02_manager_ops/03_upgrade/01_post_upgrade.yml), the appstore app update and `occ maintenance:repair --include-expensive`, only while the version `occ status` reports after `occ upgrade` differs from the one `version.php` declared before the deploy or from `files/mimetype_version`, the server version the expensive repair last ran for. +A fresh install, every upgrade path and a deploy that stopped before the repair run the pass, a redeploy of an unchanged version skips it. + +A redeploy of an unchanged version includes [`03_upgrade/02_cached_updates.yml`](./tasks/02_manager_ops/03_upgrade/02_cached_updates.yml) instead, which updates apps without querying the appstore: + +1. `occ app:list --enabled --output=json` reads the installed app versions. +2. `occ config:list updatenotification --output=json` reads the newest release per app that the daily `updatenotification` background job found. +3. `occ app:update ` runs for every enabled app whose stored release is newer than the installed one, never for an app pinned through `archive`. + +An app the background job has not checked yet is not updated until its next run. + ## MCP Server Nextcloud serves a Model Context Protocol endpoint through the AppAPI proxy of the `context_agent` ExApp. diff --git a/roles/web-app-nextcloud/filter_plugins/nextcloud_plugin_config.py b/roles/web-app-nextcloud/filter_plugins/nextcloud_plugin_config.py new file mode 100644 index 0000000000..f612f33e46 --- /dev/null +++ b/roles/web-app-nextcloud/filter_plugins/nextcloud_plugin_config.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from collections.abc import Mapping + +from ansible.plugins.filter.core import to_json + +SENSITIVE_VALUE = "***REMOVED SENSITIVE VALUE***" + + +def nextcloud_occ_value(configvalue) -> str: + """The string ``occ config:app:set --value`` receives for one entry. + + Args: + configvalue: the entry's ``configvalue`` as Ansible rendered it. + + Returns: + JSON for a mapping or a non-string sequence, ``str()`` for the rest. + """ + sequence = hasattr(configvalue, "__len__") and hasattr(configvalue, "__getitem__") + if isinstance(configvalue, Mapping) or ( + sequence and not isinstance(configvalue, str) + ): + return to_json(configvalue) + return str(configvalue) + + +def _held(current, value: str) -> bool: + if isinstance(current, bool) or current == SENSITIVE_VALUE: + return False + if isinstance(current, str): + return current == value + if isinstance(current, int): + return str(current) == value + return False + + +def _mapping(value) -> Mapping: + return value if isinstance(value, Mapping) else {} + + +def nextcloud_plugin_config_pending(specs, app_config): + """The plugin configuration entries ``occ config:app:set`` still has to write. + + Args: + specs: the addon specs of the plugins to configure, in the order their + entries are written; ``config.plugin_configuration`` is optional. + app_config: parsed ``occ config:list --output=json``. + + Returns: + list of dicts in spec and entry order, one per entry unless Nextcloud + already holds exactly its ``--value`` as a string or integer; an entry + with a ``type``, or held masked, as a boolean, float or structure, is + always returned: + appid - the app the key belongs to + configkey - the key + type - the ``--type`` of the entry, None when it has none + value - the ``--value`` string + """ + apps = _mapping(app_config["apps"]) + pending = [] + for spec in specs: + for item in (spec.get("config") or {}).get("plugin_configuration", []): + appid = item["appid"] + configkey = item["configkey"] + value = nextcloud_occ_value(item["configvalue"]) + typed = "type" in item + held = _mapping(apps.get(appid)) + if not typed and configkey in held and _held(held[configkey], value): + continue + pending.append( + { + "appid": appid, + "configkey": configkey, + "type": str(item["type"]) if typed else None, + "value": value, + } + ) + return pending + + +class FilterModule: + def filters(self): + return { + "nextcloud_plugin_config_pending": nextcloud_plugin_config_pending, + } diff --git a/roles/web-app-nextcloud/filter_plugins/nextcloud_upgrade.py b/roles/web-app-nextcloud/filter_plugins/nextcloud_upgrade.py index 36c98ef783..37ff20d9e5 100644 --- a/roles/web-app-nextcloud/filter_plugins/nextcloud_upgrade.py +++ b/roles/web-app-nextcloud/filter_plugins/nextcloud_upgrade.py @@ -11,6 +11,19 @@ _OC_VERSION = re.compile( _TAG = re.compile(r"^(?P\d+)(?:\.\d+)*(?P.*)$") _INCOMPATIBLE = re.compile(r"Disabled incompatible app: (?P[A-Za-z0-9_.-]+)") _STATUS_KEYS = ("installed", "versionstring", "maintenance", "needsDbUpgrade") +_LEADING_DIGITS = re.compile(r"[0-9]+") +_PHP_SPECIAL_FORMS = ( + ("dev", 0), + ("alpha", 1), + ("a", 1), + ("beta", 2), + ("b", 2), + ("RC", 3), + ("rc", 3), + ("#", 4), + ("pl", 5), + ("p", 5), +) def nextcloud_version_php(content): @@ -130,6 +143,149 @@ def nextcloud_status(output): return _json_object(output, _STATUS_KEYS, "nextcloud_status") +def nextcloud_needs_post_upgrade(status_output, installed_version, repaired_output): + """Return whether the running Nextcloud still owes its post-upgrade pass. + + Args: + status_output: stdout of ``occ status --output=json`` after ``occ upgrade``. + installed_version: version ``version.php`` declared before this deploy + started the pinned image, ``""`` when no data was found. + repaired_output: stdout of ``occ config:app:get files mimetype_version + --default-value=``, the server version the expensive repair last + ran for, empty when it never ran. + + Returns: + True when this deploy changed the running version or the expensive + repair has not run for it yet. + """ + running = _json_object( + status_output, (*_STATUS_KEYS, "version"), "nextcloud_needs_post_upgrade" + )["version"] + installed = str(installed_version).strip() + lines = [line.strip() for line in str(repaired_output).splitlines() if line.strip()] + repaired = lines[-1] if lines else "" + return installed not in ("", running) or repaired != running + + +def _is_digit(char): + return char != "" and char in "0123456789" + + +def _is_non_digit(char): + return not _is_digit(char) and char != "." + + +def _php_canonical_version(version): + out = [version[0]] + last = version[0] + for char in version[1:]: + if char in "-_+": + if out[-1] != ".": + out.append(".") + elif (_is_non_digit(last) and _is_digit(char)) or ( + _is_digit(last) and _is_non_digit(char) + ): + if out[-1] != ".": + out.append(".") + out.append(char) + elif not (char.isascii() and char.isalnum()): + if out[-1] != ".": + out.append(".") + else: + out.append(char) + last = char + if out[-1] == ".": + out.pop() + return "".join(out) + + +def _php_special_form(form): + for name, order in _PHP_SPECIAL_FORMS: + if form.startswith(name): + return order + return -1 + + +def _sign(value): + return (value > 0) - (value < 0) + + +def _php_compare_parts(left, right): + if _is_digit(left[:1]) and _is_digit(right[:1]): + return _sign( + int(_LEADING_DIGITS.match(left).group()) + - int(_LEADING_DIGITS.match(right).group()) + ) + if not _is_digit(left[:1]) and not _is_digit(right[:1]): + return _sign(_php_special_form(left) - _php_special_form(right)) + if _is_digit(left[:1]): + return _sign(_php_special_form("#N#") - _php_special_form(right)) + return _sign(_php_special_form(left) - _php_special_form("#N#")) + + +def _php_version_compare(left, right): + if not left or not right: + if not left and not right: + return 0 + return 1 if left else -1 + rest1 = left if left[0] == "#" else _php_canonical_version(left) + rest2 = right if right[0] == "#" else _php_canonical_version(right) + more1 = more2 = True + compare = 0 + while rest1 and rest2 and more1 and more2: + part1, dot1, tail1 = rest1.partition(".") + part2, dot2, tail2 = rest2.partition(".") + more1, more2 = bool(dot1), bool(dot2) + compare = _php_compare_parts(part1, part2) + if compare != 0: + break + if more1: + rest1 = tail1 + if more2: + rest2 = tail2 + if compare == 0: + if more1: + compare = 1 if _is_digit(rest1[:1]) else _php_version_compare(rest1, "#N#") + elif more2: + compare = -1 if _is_digit(rest2[:1]) else _php_version_compare("#N#", rest2) + return compare + + +def nextcloud_cached_app_updates(update_check_output, app_list_output, pinned): + """Return the unpinned enabled apps the daily update check found a newer release for. + + Args: + update_check_output: stdout of ``occ config:list updatenotification + --output=json``; the update check stores the newest release it saw + under the app id. + app_list_output: stdout of ``occ app:list --enabled --output=json``. + pinned: app ids whose release the role pins and never updates. + + Returns: + Sorted app ids; empty when the update check stored nothing. + """ + name = "nextcloud_cached_app_updates" + enabled = _json_object(app_list_output, ("enabled",), name)["enabled"] + stored = _json_object(update_check_output, ("apps",), name)["apps"] + if not isinstance(enabled, dict) or not isinstance(stored, dict): + return [] + if "updatenotification" not in stored: + return [] + found = stored["updatenotification"] + if not isinstance(found, dict): + return [] + updates = [] + for app, installed in enabled.items(): + if app in pinned or app not in found: + continue + available = found[app] + if not isinstance(installed, str) or not isinstance(available, str): + continue + if _php_version_compare(available.strip(), installed.strip()) > 0: + updates.append(app) + return sorted(updates) + + class FilterModule: def filters(self): return { @@ -138,4 +294,6 @@ class FilterModule: "nextcloud_major_steps": nextcloud_major_steps, "nextcloud_disabled_apps": nextcloud_disabled_apps, "nextcloud_status": nextcloud_status, + "nextcloud_needs_post_upgrade": nextcloud_needs_post_upgrade, + "nextcloud_cached_app_updates": nextcloud_cached_app_updates, } diff --git a/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade.yml b/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade.yml index cee71db62c..d7b402c086 100644 --- a/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade.yml +++ b/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade.yml @@ -2,66 +2,49 @@ - name: โฌ†๏ธ Nextcloud | Upgrade shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} upgrade" register: occ_upgrade - changed_when: "'Already up to date' not in occ_upgrade.stdout" + changed_when: "'No upgrade required' not in occ_upgrade.stdout" retries: 3 delay: 5 until: occ_upgrade.rc == 0 -- name: ๐Ÿ”ง Nextcloud | Maintenance repair - shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} maintenance:repair --include-expensive" - register: occ_repair - changed_when: "'No repairs needed' not in occ_repair.stdout" - retries: 3 - delay: 10 - until: occ_repair.rc == 0 +- name: ๐Ÿ” Nextcloud | Read the version the upgrade left running + shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} status --output=json" + register: occ_upgraded_status + changed_when: false -- name: ๐Ÿ”„ Nextcloud | App update (retry once) - shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} app:update --all" - register: occ_app_update - changed_when: "'No apps found for update' not in occ_app_update.stdout" - retries: 2 - delay: 3 - until: occ_app_update.rc == 0 - when: NEXTCLOUD_PINNED_PLUGINS | length == 0 +- name: ๐Ÿ” Nextcloud | Read the version the expensive repair last ran for + shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} config:app:get files mimetype_version --default-value=" + register: occ_repaired_version + changed_when: false -- name: ๐Ÿ”„ Nextcloud | App update that leaves the pinned releases alone - when: NEXTCLOUD_PINNED_PLUGINS | length > 0 - block: - - name: ๐Ÿ” Nextcloud | List the apps the appstore offers an update for - shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} app:update --all --showonly" - register: occ_app_updatable - changed_when: false - retries: 2 - delay: 3 - until: occ_app_updatable.rc == 0 +- name: ๐Ÿงฎ Nextcloud | Decide whether the post-upgrade pass is due + set_fact: + nextcloud_post_upgrade_due: >- + {{ occ_upgraded_status.stdout + | nextcloud_needs_post_upgrade(nextcloud_installed_version, occ_repaired_version.stdout) }} - - name: ๐Ÿ”„ Nextcloud | App update, leaving the pinned releases alone - shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} app:update {{ item | quote }}" - register: occ_app_update_unpinned - changed_when: "'No apps found for update' not in occ_app_update_unpinned.stdout" - retries: 2 - delay: 3 - until: occ_app_update_unpinned.rc == 0 - loop: >- - {{ occ_app_updatable.stdout_lines | default([]) - | select('search', ' new version available') - | map('regex_replace', ' new version available.*$', '') - | difference(NEXTCLOUD_PINNED_PLUGINS) }} +- name: ๐Ÿ”„ Nextcloud | Post-upgrade pass + ansible.builtin.include_tasks: 03_upgrade/01_post_upgrade.yml + when: nextcloud_post_upgrade_due | bool + +- name: ๐Ÿ”„ Nextcloud | Update the apps the daily update check found a release for + ansible.builtin.include_tasks: 03_upgrade/02_cached_updates.yml + when: not (nextcloud_post_upgrade_due | bool) - name: โž• Nextcloud | Add missing columns shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} db:add-missing-columns" register: occ_columns - changed_when: "'No columns found' not in occ_columns.stdout" + changed_when: "'table updated successfully' in occ_columns.stdout" - name: โž• Nextcloud | Add missing indices shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} db:add-missing-indices" register: occ_indices - changed_when: "'No indices found' not in occ_indices.stdout" + changed_when: "'table updated successfully' in occ_indices.stdout" - name: โž• Nextcloud | Add missing primary keys shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} db:add-missing-primary-keys" register: occ_pks - changed_when: "'No primary keys found' not in occ_pks.stdout" + changed_when: "'table updated successfully' in occ_pks.stdout" - name: ๐Ÿ”ด Nextcloud | Disable maintenance mode shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} maintenance:mode --off" diff --git a/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade/01_post_upgrade.yml b/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade/01_post_upgrade.yml new file mode 100644 index 0000000000..e7aea6b01c --- /dev/null +++ b/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade/01_post_upgrade.yml @@ -0,0 +1,41 @@ +--- +- name: ๐Ÿ”„ Nextcloud | App update (retry once) + shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} app:update --all" + register: occ_app_update + changed_when: "'up-to-date or no updates could be found' not in occ_app_update.stdout" + retries: 2 + delay: 3 + until: occ_app_update.rc == 0 + when: not (NEXTCLOUD_PLUGINS_PINNED | bool) + +- name: ๐Ÿ”„ Nextcloud | App update, leaving the pinned releases alone + when: NEXTCLOUD_PLUGINS_PINNED | bool + block: + - name: ๐Ÿ” Nextcloud | List the apps the appstore offers an update for + shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} app:update --all --showonly" + register: occ_app_updatable + changed_when: false + retries: 2 + delay: 3 + until: occ_app_updatable.rc == 0 + + - name: ๐Ÿ”„ Nextcloud | App update for each unpinned app the appstore offers + shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} app:update {{ item | quote }}" + register: occ_app_update_unpinned + changed_when: "'up-to-date or no updates could be found' not in occ_app_update_unpinned.stdout" + retries: 2 + delay: 3 + until: occ_app_update_unpinned.rc == 0 + loop: >- + {{ occ_app_updatable.stdout_lines + | select('search', ' new version available') + | map('regex_replace', ' new version available.*$', '') + | difference(NEXTCLOUD_PINNED_PLUGINS) }} + +- name: ๐Ÿ”ง Nextcloud | Maintenance repair + shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} maintenance:repair --include-expensive" + register: occ_repair + changed_when: true + retries: 3 + delay: 10 + until: occ_repair.rc == 0 diff --git a/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade/02_cached_updates.yml b/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade/02_cached_updates.yml new file mode 100644 index 0000000000..452e626d0b --- /dev/null +++ b/roles/web-app-nextcloud/tasks/02_manager_ops/03_upgrade/02_cached_updates.yml @@ -0,0 +1,21 @@ +--- +- name: ๐Ÿ“‹ Nextcloud | Read the installed app versions + shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} app:list --enabled --output=json" + register: occ_enabled_apps + changed_when: false + +- name: ๐Ÿ“‹ Nextcloud | Read the app releases the daily update check found + shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} config:list updatenotification --output=json" + register: occ_update_check + changed_when: false + +- name: ๐Ÿ”„ Nextcloud | App update for the releases the daily update check found + shell: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} app:update {{ item | quote }}" + register: occ_app_update_cached + changed_when: "'up-to-date or no updates could be found' not in occ_app_update_cached.stdout" + retries: 2 + delay: 3 + until: occ_app_update_cached.rc == 0 + loop: >- + {{ occ_update_check.stdout + | nextcloud_cached_app_updates(occ_enabled_apps.stdout, NEXTCLOUD_PINNED_PLUGINS) }} diff --git a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin.yml b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin.yml index ef38c302f8..8fcaff427d 100644 --- a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin.yml +++ b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin.yml @@ -8,6 +8,7 @@ - name: "๐Ÿงฎ Plan which plugins need an install, an enable or only their config" ansible.builtin.set_fact: nextcloud_plugin_plan: "{{ NEXTCLOUD_PLUGIN_ITEMS | nextcloud_plugin_plan(nextcloud_app_list.stdout | from_json) }}" + nextcloud_plugin_configure_keys: [] - name: "๐Ÿ”ด Disable the plugins that end up disabled" ansible.builtin.command: @@ -15,7 +16,7 @@ when: nextcloud_plugin_plan.disable | length > 0 changed_when: true -- name: "๐Ÿ“ฆ Install, enable and configure the plugins Nextcloud lacks" +- name: "๐Ÿ“ฆ Install and enable the plugins Nextcloud lacks" ansible.builtin.include_tasks: 06_plugin/01_install.yml loop: "{{ NEXTCLOUD_PLUGIN_ITEMS | selectattr('key', 'in', nextcloud_plugin_plan.install) | list }}" loop_control: @@ -26,7 +27,7 @@ plugin_key: "{{ plugin_item.key }}" plugin_value: "{{ plugin_item.value }}" -- name: "๐ŸŸข Enable and configure the installed plugins Nextcloud keeps disabled" +- name: "๐ŸŸข Enable the installed plugins Nextcloud keeps disabled" ansible.builtin.include_tasks: 06_plugin/02_enable.yml loop: "{{ NEXTCLOUD_PLUGIN_ITEMS | selectattr('key', 'in', nextcloud_plugin_plan.enable) | list }}" loop_control: @@ -36,16 +37,11 @@ vars: plugin_key: "{{ plugin_item.key }}" plugin_value: "{{ plugin_item.value }}" - plugin_task_path: "{{ NEXTCLOUD_CNODE_PLUGIN_TASKS_PATH }}{{ plugin_item.key }}.yml" -- name: "๐Ÿ”ง Configure the plugins Nextcloud already runs" +- name: "๐Ÿงฎ Queue the plugins Nextcloud already runs after the enabled ones" + ansible.builtin.set_fact: + nextcloud_plugin_configure_keys: "{{ nextcloud_plugin_configure_keys + nextcloud_plugin_plan.configure }}" + +- name: "๐Ÿ”ง Configure every queued plugin in one batch" ansible.builtin.include_tasks: 06_plugin/03_configure.yml - loop: "{{ NEXTCLOUD_PLUGIN_ITEMS | selectattr('key', 'in', nextcloud_plugin_plan.configure) | list }}" - loop_control: - loop_var: plugin_item - label: "{{ plugin_item.key }}" - no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" - vars: - plugin_key: "{{ plugin_item.key }}" - plugin_value: "{{ plugin_item.value }}" - plugin_task_path: "{{ NEXTCLOUD_CNODE_PLUGIN_TASKS_PATH }}{{ plugin_item.key }}.yml" + when: nextcloud_plugin_configure_keys | length > 0 diff --git a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/01_install.yml b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/01_install.yml index d431726715..db1785ee09 100644 --- a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/01_install.yml +++ b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/01_install.yml @@ -81,5 +81,3 @@ - include_tasks: 02_enable.yml when: (install_result | nextcloud_install_status).runnable - vars: - plugin_task_path: "{{ NEXTCLOUD_CNODE_PLUGIN_TASKS_PATH }}{{ plugin_key }}.yml" diff --git a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/02_enable.yml b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/02_enable.yml index ed7911a55c..385adc71ca 100644 --- a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/02_enable.yml +++ b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/02_enable.yml @@ -20,6 +20,7 @@ currently pinned Nextcloud server version. Skipping configure for this run. when: plugin_enable_incompatible -- name: "๐Ÿ”ง Configure {{ plugin_key }} (skipped when incompatible)" - ansible.builtin.include_tasks: 03_configure.yml +- name: "๐Ÿงฎ Queue {{ plugin_key }} for the batched configuration" + ansible.builtin.set_fact: + nextcloud_plugin_configure_keys: "{{ nextcloud_plugin_configure_keys + [plugin_key] }}" when: not plugin_enable_incompatible diff --git a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/03_configure.yml b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/03_configure.yml index cd3c7802c4..df567eb4ad 100644 --- a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/03_configure.yml +++ b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/03_configure.yml @@ -1,34 +1,46 @@ --- # Loaded via include_vars (not config lookup or lookup('file')) so the role-scoped vars referenced inside the payload render lazily in this task's scope. -- name: "๐Ÿ“„ Load {{ plugin_key }} addon spec from meta/addons/{{ plugin_key }}.yml" +- name: "๐Ÿ“„ Load the addon specs of the plugins to configure from meta/addons/" ansible.builtin.include_vars: file: "{{ [role_path, 'meta/addons', plugin_key ~ '.yml'] | path_join }}" - name: addon_spec + name: "nextcloud_addon_spec_{{ plugin_key }}" + loop: "{{ nextcloud_plugin_configure_keys }}" + loop_control: + loop_var: plugin_key -- name: "๐Ÿ”ง Set plugin configuration (one exec, occ calls on stdin)" +- name: "๐Ÿ“‹ Read the app configuration Nextcloud holds" + ansible.builtin.command: + cmd: "{{ NEXTCLOUD_DOCKER_EXEC_OCC }} config:list --output=json" + register: nextcloud_app_config + changed_when: false + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "๐Ÿงฎ Keep the plugin configuration values Nextcloud does not hold yet" + ansible.builtin.set_fact: + nextcloud_plugin_config_pending: >- + {{ query('vars', *(nextcloud_plugin_configure_keys | map('regex_replace', '^', 'nextcloud_addon_spec_'))) + | nextcloud_plugin_config_pending(nextcloud_app_config.stdout | from_json) }} + no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" + +- name: "๐Ÿ”ง Set the pending plugin configuration (one exec, occ calls on stdin)" ansible.builtin.shell: cmd: "{{ NEXTCLOUD_DOCKER_EXEC_INTERACTIVE }} sh -s" stdin: | set -eu - {% for item in (addon_spec.config.plugin_configuration | default([])) %} - {{ NEXTCLOUD_INTERNAL_OCC_COMMAND | quote }} \ - config:app:set {{ item.appid }} {{ item.configkey }}{{ (' --no-interaction --type=' ~ item.type) if item.type is defined else '' }} \ - --value '{{ ( (item.configvalue | to_json) if (item.configvalue is mapping or (item.configvalue is sequence and item.configvalue is not string)) else (item.configvalue | string) ) - | regex_replace("'", "'" ~ '"' ~ "'" ~ '"' ~ "'") }}' + {% for item in nextcloud_plugin_config_pending %} + {{ NEXTCLOUD_INTERNAL_OCC_COMMAND | quote }} config:app:set {{ item.appid | quote }} {{ item.configkey | quote }}{{ (' --no-interaction --type=' ~ item.type) if item.type is not none else '' }} --value {{ item.value | quote }} || { echo {{ ('config:app:set failed for ' ~ item.appid ~ '/' ~ item.configkey) | quote }} >&2; exit 1; } {% endfor %} - when: (addon_spec.config.plugin_configuration | default([])) | length > 0 - register: config_set_shell - changed_when: config_set_shell.stdout is search(' set to ') - failed_when: config_set_shell.rc != 0 + when: nextcloud_plugin_config_pending | length > 0 + register: nextcloud_plugin_config_set + changed_when: nextcloud_plugin_config_set.stdout is search(' set to ') + failed_when: nextcloud_plugin_config_set.rc != 0 no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}" -- name: ๐Ÿ”Ž Check if {{ plugin_task_path }} exists - stat: - path: "{{ plugin_task_path }}" - delegate_to: localhost - become: false - register: plugin_tasks_file - -- name: "๐Ÿ“‚ include {{ plugin_task_path }}" - include_tasks: "{{ plugin_task_path }}" - when: plugin_tasks_file.stat.exists +- name: "๐Ÿ“‚ Run the hook of every configured plugin from tasks/addons/" + ansible.builtin.include_tasks: "{{ plugin_task_path }}" + loop: "{{ nextcloud_plugin_configure_keys }}" + loop_control: + loop_var: plugin_key + vars: + plugin_task_path: "{{ NEXTCLOUD_CNODE_PLUGIN_TASKS_PATH }}{{ plugin_key }}.yml" + when: plugin_task_path is file diff --git a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/README.md b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/README.md index fe5d9a1bae..ef90610062 100644 --- a/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/README.md +++ b/roles/web-app-nextcloud/tasks/02_manager_ops/06_plugin/README.md @@ -28,8 +28,19 @@ Each `tasks/02_manager_ops/06_plugin/.yml` file owns exactly one OCC stage | File | Stage | OCC command | Continues to next stage when | |---|---|---|---| | [01_install.yml](./01_install.yml) | install | `occ app:install` | install result is `runnable` (success or already installed) | -| [02_enable.yml](./02_enable.yml) | enable | `occ app:enable` | enable result is not flagged `incompatible` | -| [03_configure.yml](./03_configure.yml) | configure | `occ config:app:set` plus optional hook | (terminal stage) | +| [02_enable.yml](./02_enable.yml) | enable | `occ app:enable` | enable result is not flagged `incompatible`; the plugin is then queued for the configure stage | +| [03_configure.yml](./03_configure.yml) | configure | one `occ config:list`, then `occ config:app:set` for the values that differ, plus optional hook | (terminal stage) | + +## Configure stage + +[03_configure.yml](./03_configure.yml) runs once for every queued plugin: first the plugins the install loop enabled, then those the enable loop enabled, then those of the `configure` plan list. + +1. It loads the addon spec of every queued plugin. +2. It reads `occ config:list` once and keeps an entry only when Nextcloud does not already hold the exact value `occ config:app:set` would store. Entries with a `type`, sensitive values that `config:list` masks, booleans, floats and structured values are always kept. +3. It writes the kept entries in spec order through one shell; the first failing `config:app:set` stops the shell and names its `appid/configkey` on stderr. The task reports changed only when `occ` answers `set to`. +4. It runs the hook of every queued plugin in queue order. + +The keep or drop decision lives in the filter `nextcloud_plugin_config_pending` ([filter_plugins/nextcloud_plugin_config.py](../../../filter_plugins/nextcloud_plugin_config.py)). The classification of "runnable" and "incompatible" is computed by the Jinja filter `nextcloud_install_status` defined in [roles/web-app-nextcloud/filter_plugins/nextcloud_install.py](../../../filter_plugins/nextcloud_install.py). The filter inspects `rc`, `stdout`, and `stderr` from the OCC call and returns flags that drive `until`, `failed_when`, and `when`. @@ -60,10 +71,10 @@ A new plugin requires no changes here when its addon spec and optional hook file - **Configuration keys (optional).** Add a `config.plugin_configuration` list to `meta/addons/.yml`. Each list item MUST contain `appid`, `configkey`, and `configvalue`. - `03_configure.yml` loads the addon spec and invokes `occ config:app:set` for every entry. + `03_configure.yml` loads the addon spec and invokes `occ config:app:set` for every entry whose value Nextcloud does not hold yet. - **Custom logic (optional).** Create `tasks/addons/.yml`. - The hook runs after `config:app:set` and MAY perform plugin-specific provisioning such as LDAP wiring or REST calls against the live container. + The hook runs after the configuration of every queued plugin and MAY perform plugin-specific provisioning such as LDAP wiring or REST calls against the live container. The hook is looked up via the path constant `NEXTCLOUD_CNODE_PLUGIN_TASKS_PATH` defined in [vars/main.yml](../../../vars/main.yml). Missing files MUST be tolerated; the corresponding step skips cleanly. @@ -77,4 +88,4 @@ This guarantees that mutually exclusive integrations such as the OIDC entry poin ## Entry point [02_manager_ops.yml](../../02_manager_ops.yml) includes [06_plugin.yml](../06_plugin.yml) once. -It loops over the `NEXTCLOUD_PLUGIN_ITEMS` of each plan list and unpacks the loop variable `plugin_item` into `plugin_key` (the OCC app name) and `plugin_value` (its meta entry). +It loops over the `NEXTCLOUD_PLUGIN_ITEMS` of the `install` and `enable` plan lists and unpacks the loop variable `plugin_item` into `plugin_key` (the OCC app name) and `plugin_value` (its meta entry), then includes the configure stage once for the queue `nextcloud_plugin_configure_keys`. diff --git a/roles/web-app-nextcloud/vars/main.yml b/roles/web-app-nextcloud/vars/main.yml index 5d3e451cf3..1546fcff2a 100644 --- a/roles/web-app-nextcloud/vars/main.yml +++ b/roles/web-app-nextcloud/vars/main.yml @@ -22,6 +22,7 @@ NEXTCLOUD_PINNED_PLUGINS: >- | selectattr('value.config.archive', 'defined') | selectattr('value.enabled', 'true') | map(attribute='key') | list }} +NEXTCLOUD_PLUGINS_PINNED: "{{ NEXTCLOUD_PINNED_PLUGINS | length > 0 }}" NEXTCLOUD_PLUGINS_ENABLED: "{{ lookup('config', application_id, 'services.nextcloud.plugins_enabled') }}" NEXTCLOUD_DESIGN_ENABLED: "{{ lookup('config', application_id, 'services.design.enabled') | bool }}" NEXTCLOUD_DESIGN_DIR: "{{ [lookup('container', application_id, 'directories.config'), 'design'] | path_join }}" diff --git a/roles/web-app-stalwart/meta/services.yml b/roles/web-app-stalwart/meta/services.yml index c499a79e0a..0aa654445e 100644 --- a/roles/web-app-stalwart/meta/services.yml +++ b/roles/web-app-stalwart/meta/services.yml @@ -19,6 +19,7 @@ dashboard: bond: 1 enabled: "{{ 'web-app-dashboard' in group_names }}" shared: "{{ 'web-app-dashboard' in group_names }}" + card: webmail matomo: bond: 1 enabled: "{{ 'web-app-matomo' in group_names }}" diff --git a/roles/web-app-xwiki/tasks/00_core.yml b/roles/web-app-xwiki/tasks/00_core.yml index 255105958b..dc71d7fa48 100644 --- a/roles/web-app-xwiki/tasks/00_core.yml +++ b/roles/web-app-xwiki/tasks/00_core.yml @@ -4,6 +4,9 @@ - name: โœ… Validate XWiki variables include_tasks: 01_validation.yml +- name: "๐Ÿ”Ž Probe whether the running XWiki already carries every add-on on its pin" + include_tasks: utils/extensions_ready.yml + - block: - name: "๐Ÿงฉ load docker, db and proxy for {{ application_id }}" include_role: @@ -26,8 +29,8 @@ - name: ๐Ÿš€ Deploy Bootstrap Config include_tasks: utils/flush_config.yml vars: - xwiki_oidc_enabled_switch: false - xwiki_ldap_enabled_switch: false + xwiki_oidc_enabled_switch: "{{ (XWIKI_OIDC_ENABLED | bool) and (xwiki_extensions_ready | bool) }}" + xwiki_ldap_enabled_switch: "{{ (XWIKI_LDAP_ENABLED | bool) and (xwiki_extensions_ready | bool) }}" xwiki_blobstore_s3_enabled_switch: false - name: "๐Ÿ‘” XWiki manager-side operations (file deploys + container exec)" diff --git a/roles/web-app-xwiki/tasks/04_extensions.yml b/roles/web-app-xwiki/tasks/04_extensions.yml index 75d9d8a3be..7b8822602d 100644 --- a/roles/web-app-xwiki/tasks/04_extensions.yml +++ b/roles/web-app-xwiki/tasks/04_extensions.yml @@ -3,11 +3,6 @@ # Extension Script Service (services.extension.install). # Avoids REST job API and any Namespace class import for portability. # -# Flow: -# - Bootstrap config renders with both auth backends OFF (already in your role). -# - This file installs required extensions on the current wiki. -# - Final config later turns auth ON (already in your role). -# # Notes: # - We print machine-readable markers so Ansible can assert deterministically. diff --git a/roles/web-app-xwiki/tasks/utils/extensions_ready.yml b/roles/web-app-xwiki/tasks/utils/extensions_ready.yml new file mode 100644 index 0000000000..780bdb43bd --- /dev/null +++ b/roles/web-app-xwiki/tasks/utils/extensions_ready.yml @@ -0,0 +1,67 @@ +--- +- name: "๐Ÿ”Ž XWIKI | Probe whether the running wiki answers the superadmin" + uri_retry: + url: "{{ [XWIKI_REST_XWIKI, 'spaces'] | url_join }}" + method: GET + url_username: "{{ XWIKI_SUPERADMIN_USERNAME }}" + url_password: "{{ XWIKI_SUPERADMIN_PASSWORD }}" + force_basic_auth: true + follow_redirects: none + return_content: false + status_code: "{{ [-1] + range(100, 600) | list }}" + register: _xwiki_running + changed_when: false + when: + - IS_STACK_HOST | bool + - XWIKI_SSO_ENABLED | bool + +- name: "๐Ÿ”Ž XWIKI | Read the core version of the running wiki" + uri_retry: + url: "{{ XWIKI_REST_BASE }}" + method: GET + url_username: "{{ XWIKI_SUPERADMIN_USERNAME }}" + url_password: "{{ XWIKI_SUPERADMIN_PASSWORD }}" + force_basic_auth: true + follow_redirects: none + headers: + Accept: "application/xml" + return_content: true + status_code: "{{ [-1] + range(100, 600) | list }}" + register: _xwiki_running_core + changed_when: false + when: + - _xwiki_running is not skipped + - _xwiki_running.status == 200 + +- name: "๐Ÿงญ XWIKI | Compare the running core with the pinned core {{ XWIKI_CORE_VERSION }}" + set_fact: + _xwiki_core_on_pin: >- + {{ _xwiki_running_core is not skipped + and _xwiki_running_core.status == 200 + and _xwiki_running_core.content is search('' ~ (XWIKI_CORE_VERSION | regex_escape) ~ '') }} + +- name: "โœ… XWIKI | Check the running wiki's add-ons against their pins" + include_tasks: "{{ role_path }}/tasks/_check_extension_via_groovy.yml" + loop: "{{ XWIKI_PLUGINS }}" + loop_control: + loop_var: plugin + label: "{{ plugin.id }}" + vars: + ext_id: "{{ plugin.id }}" + ext_version: "{{ plugin.version }}" + result_var: "xwiki_running_{{ plugin.id | regex_replace('[^A-Za-z0-9_]', '_') }}" + when: _xwiki_core_on_pin | bool + +- name: "๐Ÿงญ XWIKI | Skip the bootstrap boot when core and add-ons run on their pins" + set_fact: + xwiki_extensions_ready: >- + {{ (_xwiki_core_on_pin | bool) + and (XWIKI_PLUGINS + | map(attribute='id') + | map('regex_replace', '[^A-Za-z0-9_]', '_') + | map('regex_replace', '^', 'xwiki_running_') + | map('extract', hostvars[inventory_hostname], 'status') + | map('int') + | reject('equalto', 200) + | list + | length == 0) }} diff --git a/roles/web-app-xwiki/templates/style.css.j2 b/roles/web-app-xwiki/templates/style.css.j2 index 274d66405a..d8c5531241 100644 --- a/roles/web-app-xwiki/templates/style.css.j2 +++ b/roles/web-app-xwiki/templates/style.css.j2 @@ -4045,7 +4045,7 @@ ul.xlist li.showAllResults button.xitemcontainer { } .jstree-xwiki .jstree-node, .jstree-xwiki .jstree-icon { - background-image: url("{{ XWIKI_URL }}/webjars/wiki%3Axwiki/xwiki-platform-tree-webjar/{{ XWIKI_PLATFORM_VERSION }}/32px.png"); + background-image: url("{{ XWIKI_URL }}/webjars/wiki%3Axwiki/xwiki-platform-tree-webjar/{{ XWIKI_CORE_VERSION }}/32px.png"); } .jstree-xwiki .jstree-last { background-image: none; @@ -4067,10 +4067,10 @@ ul.xlist li.showAllResults button.xitemcontainer { background-image: none; } .jstree-xwiki .jstree-file { - background: url("{{ XWIKI_URL }}/webjars/wiki%3Axwiki/xwiki-platform-tree-webjar/{{ XWIKI_PLATFORM_VERSION }}/32px.png") -100px -68px no-repeat; + background: url("{{ XWIKI_URL }}/webjars/wiki%3Axwiki/xwiki-platform-tree-webjar/{{ XWIKI_CORE_VERSION }}/32px.png") -100px -68px no-repeat; } .jstree-xwiki .jstree-folder { - background: url("{{ XWIKI_URL }}/webjars/wiki%3Axwiki/xwiki-platform-tree-webjar/{{ XWIKI_PLATFORM_VERSION }}/32px.png") -260px -4px no-repeat; + background: url("{{ XWIKI_URL }}/webjars/wiki%3Axwiki/xwiki-platform-tree-webjar/{{ XWIKI_CORE_VERSION }}/32px.png") -260px -4px no-repeat; } .jstree-xwiki-small .jstree-disabled.jstree-hovered { background-color: transparent; @@ -4114,10 +4114,10 @@ ul.xlist li.showAllResults button.xitemcontainer { background-color: transparent; } .jstree-xwiki-responsive .jstree-file { - background: url("{{ XWIKI_URL }}/webjars/wiki%3Axwiki/xwiki-platform-tree-webjar/{{ XWIKI_PLATFORM_VERSION }}/40px.png") 0 -160px no-repeat; + background: url("{{ XWIKI_URL }}/webjars/wiki%3Axwiki/xwiki-platform-tree-webjar/{{ XWIKI_CORE_VERSION }}/40px.png") 0 -160px no-repeat; } .jstree-xwiki-responsive .jstree-folder { - background: url("{{ XWIKI_URL }}/webjars/wiki%3Axwiki/xwiki-platform-tree-webjar/{{ XWIKI_PLATFORM_VERSION }}/40px.png") -40px -40px no-repeat; + background: url("{{ XWIKI_URL }}/webjars/wiki%3Axwiki/xwiki-platform-tree-webjar/{{ XWIKI_CORE_VERSION }}/40px.png") -40px -40px no-repeat; } } .jstree-xwiki>.jstree-no-dots.jstree-no-icons .jstree-ocl { diff --git a/roles/web-app-xwiki/vars/main.yml b/roles/web-app-xwiki/vars/main.yml index 29b5784936..0ef6054c66 100644 --- a/roles/web-app-xwiki/vars/main.yml +++ b/roles/web-app-xwiki/vars/main.yml @@ -15,7 +15,7 @@ XWIKI_DOCK_DATA_DIR: "{{ lookup('volume', 'web-app-xwiki', 'data').target }}" XWIKI_IMAGE: "{{ lookup('config', application_id, 'services.xwiki.image') }}" XWIKI_VERSION: "{{ lookup('config', application_id, 'services.xwiki.version') }}" -XWIKI_PLATFORM_VERSION: "{{ XWIKI_VERSION | split('-') | first }}" +XWIKI_CORE_VERSION: "{{ (XWIKI_VERSION | string).split('-') | first }}" XWIKI_CONTAINER: "{{ lookup('config', application_id, 'services.xwiki.name') }}" XWIKI_CONTAINER_ADDRESS: "{{ lookup('container_address', application_id, entity_name) }}" diff --git a/roles/web-app-yourls/files/php/upgrade_database.php b/roles/web-app-yourls/files/php/upgrade_database.php new file mode 100644 index 0000000000..0a51a374a5 --- /dev/null +++ b/roles/web-app-yourls/files/php/upgrade_database.php @@ -0,0 +1,72 @@ + -> " or "current ", exit 0. + * Failure: diagnostics on stderr, exit 1. Usage error: exit 2. + */ + +if ($argc !== 2) { + fwrite(STDERR, "usage: php upgrade_database.php YOURLS_ROOT\n"); + exit(2); +} + +define('YOURLS_UPGRADING', true); + +$infinito_upgrade_done = false; + +// Exception: YOURLS reports bootstrap and upgrade errors with die(), which exits 0; only this handler turns them into a failure. +register_shutdown_function(static function (): void { + if ($GLOBALS['infinito_upgrade_done']) { + return; + } + $chatter = ''; + while (ob_get_level() > 0) { + $chatter = ob_get_clean() . $chatter; + } + fwrite(STDERR, "YOURLS stopped before the database upgrade completed\n" . trim(strip_tags($chatter)) . "\n"); + exit(1); +}); + +function infinito_upgrade_report(string $line): void +{ + $GLOBALS['infinito_upgrade_done'] = true; + while (ob_get_level() > 0) { + ob_end_clean(); + } + echo $line, "\n"; + exit(0); +} + +function infinito_upgrade_fail(string $reason): void +{ + $GLOBALS['infinito_upgrade_done'] = true; + $chatter = ''; + while (ob_get_level() > 0) { + $chatter = ob_get_clean() . $chatter; + } + fwrite(STDERR, $reason . "\n" . trim(strip_tags($chatter)) . "\n"); + exit(1); +} + +ob_start(); +require rtrim($argv[1], '/') . '/includes/load-yourls.php'; + +list($infinito_old_version, $infinito_old_sql) = yourls_get_current_version_from_sql(); + +if (!yourls_upgrade_is_needed()) { + infinito_upgrade_report('current ' . $infinito_old_sql); +} + +if ((int) $infinito_old_sql < 200) { + infinito_upgrade_fail('db_version ' . $infinito_old_sql . ' predates YOURLS 1.4; its multi-request row conversion needs admin/upgrade.php'); +} + +yourls_upgrade(1, $infinito_old_version, YOURLS_VERSION, $infinito_old_sql, YOURLS_DB_VERSION); +yourls_upgrade(3, $infinito_old_version, YOURLS_VERSION, $infinito_old_sql, YOURLS_DB_VERSION); + +if (yourls_upgrade_is_needed()) { + infinito_upgrade_fail('YOURLS still needs a database upgrade after running it'); +} + +infinito_upgrade_report('upgraded ' . $infinito_old_sql . ' -> ' . YOURLS_DB_VERSION); diff --git a/roles/web-app-yourls/tasks/install.yml b/roles/web-app-yourls/tasks/install.yml index 859cb79958..7b971426c3 100644 --- a/roles/web-app-yourls/tasks/install.yml +++ b/roles/web-app-yourls/tasks/install.yml @@ -27,3 +27,9 @@ changed_when: "'tables successfully created' in yourls_install.stdout" failed_when: yourls_install.rc != 0 delegate_to: "{{ YOURLS_HOST_NODE }}" + +- name: "โฌ†๏ธ Bring the YOURLS database schema up to the running code" + ansible.builtin.include_tasks: + file: upgrade.yml + apply: + delegate_to: "{{ YOURLS_HOST_NODE }}" diff --git a/roles/web-app-yourls/tasks/main.yml b/roles/web-app-yourls/tasks/main.yml index a2e90ccd79..7be8882852 100644 --- a/roles/web-app-yourls/tasks/main.yml +++ b/roles/web-app-yourls/tasks/main.yml @@ -19,6 +19,6 @@ - name: "๐Ÿšฟ Flush compose handlers for {{ application_id }}" ansible.builtin.meta: flush_handlers -- name: "๐Ÿ“ฆ Install the YOURLS database schema for {{ application_id }}" +- name: "๐Ÿ“ฆ Install and upgrade the YOURLS database schema for {{ application_id }}" ansible.builtin.include_tasks: install.yml when: IS_STACK_HOST | bool diff --git a/roles/web-app-yourls/tasks/upgrade.yml b/roles/web-app-yourls/tasks/upgrade.yml new file mode 100644 index 0000000000..902930cfc6 --- /dev/null +++ b/roles/web-app-yourls/tasks/upgrade.yml @@ -0,0 +1,23 @@ +--- +- name: "โฌ†๏ธ Upgrade the YOURLS database schema when the code is ahead of it" + ansible.builtin.shell: + cmd: >- # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec -i --user {{ YOURLS_WEB_USER | quote }} {{ YOURLS_LOCAL_CID | quote }} + php -- {{ YOURLS_DOCROOT | quote }} + stdin: "{{ lookup('file', 'php/upgrade_database.php') }}" + register: yourls_upgrade + changed_when: yourls_upgrade.stdout is match('upgraded ') + failed_when: yourls_upgrade.rc != 0 or yourls_upgrade.stdout is not match('(upgraded|current) ') + +- name: "๐Ÿ”Ž Verify a short URL no longer redirects to admin/upgrade.php" + ansible.builtin.shell: + cmd: >- # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec {{ YOURLS_LOCAL_CID | quote }} + {{ 30 | curl }} -o /dev/null -w '%{http_code} %{redirect_url}' + http://127.0.0.1:{{ container_port }}/{{ YOURLS_UPGRADE_PROBE_KEYWORD }} + register: yourls_short_url_probe + changed_when: false + failed_when: >- + yourls_short_url_probe.rc != 0 + or yourls_short_url_probe.stdout is not match('30[12] ') + or 'admin/upgrade.php' in yourls_short_url_probe.stdout diff --git a/roles/web-app-yourls/vars/main.yml b/roles/web-app-yourls/vars/main.yml index 5a34d50263..0405654f05 100644 --- a/roles/web-app-yourls/vars/main.yml +++ b/roles/web-app-yourls/vars/main.yml @@ -13,6 +13,9 @@ YOURLS_CONTAINER_ADDRESS: "{{ lookup('container_address', application_id, entity YOURLS_URL: "{{ lookup('tls', application_id, 'url.base') | regex_replace('/+$', '') }}" YOURLS_ADMIN_LOCATION: "{{ lookup('config', application_id, 'server.locations.admin') }}" YOURLS_LANDINGPAGE_STATUS_CODE: "{{ lookup('config', application_id, 'server.status_codes.default') }}" +YOURLS_DOCROOT: "/var/www/html" +YOURLS_WEB_USER: "www-data" +YOURLS_UPGRADE_PROBE_KEYWORD: "infinito-upgrade-probe" YOURLS_SSO_PROXY_GATED: "{{ lookup('sso', application_id, 'is_proxy_gated') }}" YOURLS_SSO_ADMIN_GROUP: "/{{ lookup('rbac_group_path', application_id=application_id, role='administrator') }}" diff --git a/tests/integration/ansible/filters/test_defined.py b/tests/integration/ansible/filters/test_defined.py index 34761f14a7..c98d537137 100644 --- a/tests/integration/ansible/filters/test_defined.py +++ b/tests/integration/ansible/filters/test_defined.py @@ -68,6 +68,7 @@ BUILTIN_FILTERS: set[str] = { "b64encode", "basename", "dirname", + "relpath", "from_json", "to_json", "from_yaml", diff --git a/tests/lint/ansible/roles/meta/test_dashboard_card_key.py b/tests/lint/ansible/roles/meta/test_dashboard_card_key.py new file mode 100644 index 0000000000..6e393166b6 --- /dev/null +++ b/tests/lint/ansible/roles/meta/test_dashboard_card_key.py @@ -0,0 +1,83 @@ +"""Lint guard for the dashboard card target a role names in ``meta/services.yml``. + +A role MAY point its dashboard card at a non-primary domain by naming +``card: `` on its ``dashboard`` binding. The key MUST be a key of the +role's ``meta/domains.yml`` ``canonical`` mapping, the only keys +``lookup('canonical_url', , )`` resolves. +""" + +from __future__ import annotations + +import unittest +from typing import TYPE_CHECKING, Any + +import yaml +from ansible.errors import AnsibleError + +from utils.cache.files import read_text +from utils.cache.yaml import load_yaml_str +from utils.roles.applications.dashboard import ( + CARD_TARGET_KEY, + DASHBOARD_SERVICE, + card_target_key, +) +from utils.roles.mapping import ROLE_FILE_META_DOMAINS, ROLE_FILE_META_SERVICES + +from . import PROJECT_ROOT + +if TYPE_CHECKING: + from pathlib import Path + +ROLES_DIR = PROJECT_ROOT / "roles" + + +def _load_yaml(path: Path) -> Any: + if not path.is_file(): + return None + try: + return load_yaml_str(read_text(str(path))) + except (UnicodeDecodeError, yaml.YAMLError): + return None + + +def _card_key_violation(role_dir: Path) -> str | None: + role_name = role_dir.name + services = _load_yaml(role_dir / ROLE_FILE_META_SERVICES) + if not isinstance(services, dict): + return None + try: + key = card_target_key(role_name, services.get(DASHBOARD_SERVICE)) + except AnsibleError as exc: + return str(exc) + if not key: + return None + domains = _load_yaml(role_dir / ROLE_FILE_META_DOMAINS) + canonical = domains.get("canonical") if isinstance(domains, dict) else None + if not isinstance(canonical, dict) or key not in canonical: + available = sorted(canonical) if isinstance(canonical, dict) else [] + return ( + f"{role_name}: services.{DASHBOARD_SERVICE}.{CARD_TARGET_KEY} '{key}' " + f"is not a key of {ROLE_FILE_META_DOMAINS} canonical " + f"(available: {available})." + ) + return None + + +class TestDashboardCardKey(unittest.TestCase): + def test_card_key_is_a_domain_key_of_its_role(self) -> None: + violations = [ + violation + for role_dir in sorted(ROLES_DIR.iterdir()) + if role_dir.is_dir() + and (violation := _card_key_violation(role_dir)) is not None + ] + if violations: + self.fail( + f"A dashboard binding's `{CARD_TARGET_KEY}` MUST name a " + "domains.canonical key of its own role:\n" + + "\n".join(f" - {v}" for v in violations) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/php/roles/web-app-yourls/files/UpgradeDatabaseTest.php b/tests/unit/php/roles/web-app-yourls/files/UpgradeDatabaseTest.php new file mode 100644 index 0000000000..7b58b27a60 --- /dev/null +++ b/tests/unit/php/roles/web-app-yourls/files/UpgradeDatabaseTest.php @@ -0,0 +1,199 @@ +Incorrect DB config, or could not connect to DB

'; + die(); + } + define('YOURLS_VERSION', '1.10.6'); + define('YOURLS_DB_VERSION', '507'); + + function yourls_get_current_version_from_sql() + { + $state = json_decode((string) file_get_contents(getenv('STUB_STATE')), true); + return [$state['version'], $state['db_version']]; + } + + function yourls_upgrade_is_needed() + { + list(, $sql) = yourls_get_current_version_from_sql(); + return $sql < YOURLS_DB_VERSION; + } + + function yourls_upgrade($step, $oldver, $newver, $oldsql, $newsql) + { + file_put_contents(getenv('STUB_LOG'), "$step $oldver $newver $oldsql $newsql\n", FILE_APPEND); + echo '

Updating DB. Please wait...

'; + if (getenv('STUB_FAIL_STEP') === (string) $step) { + echo "

Unable to update the DB.

"; + die(); + } + if ($step === 3 && getenv('STUB_SKIP_SAVE') !== '1') { + file_put_contents(getenv('STUB_STATE'), json_encode(['version' => $newver, 'db_version' => $newsql])); + } + } + PHP; + + private string $root; + + private string $state; + + private string $log; + + protected function setUp(): void + { + $this->root = sys_get_temp_dir() . '/yourls-upgrade-' . bin2hex(random_bytes(6)); + mkdir($this->root . '/includes', 0700, true); + file_put_contents($this->root . '/includes/load-yourls.php', self::STUB); + $this->state = $this->root . '/state.json'; + $this->log = $this->root . '/upgrade.log'; + touch($this->log); + } + + protected function tearDown(): void + { + foreach ([$this->root . '/includes/load-yourls.php', $this->state, $this->log] as $file) { + if (is_file($file)) { + unlink($file); + } + } + rmdir($this->root . '/includes'); + rmdir($this->root); + } + + private function seed(string $version, string $dbVersion): void + { + file_put_contents($this->state, json_encode(['version' => $version, 'db_version' => $dbVersion])); + } + + /** + * Run the script against the stub YOURLS root. + * + * @param array $env extra environment for the child + * @param list|null $args script arguments, null for the stub root + * @return array{0: string, 1: string, 2: int} stdout, stderr and exit code + */ + private function runScript(array $env = [], ?array $args = null): array + { + $script = dirname(__DIR__, 6) . '/roles/web-app-yourls/files/php/upgrade_database.php'; + $process = proc_open( + array_merge([PHP_BINARY, $script], $args ?? [$this->root]), + [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], + $pipes, + null, + $env + [ + 'STUB_STATE' => $this->state, + 'STUB_LOG' => $this->log, + 'PATH' => (string) getenv('PATH'), + ] + ); + $stdout = (string) stream_get_contents($pipes[1]); + $stderr = (string) stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + + return [$stdout, $stderr, proc_close($process)]; + } + + private function upgradeCalls(): string + { + return (string) file_get_contents($this->log); + } + + public function testCurrentSchemaIsReportedWithoutUpgrading(): void + { + $this->seed('1.10.6', '507'); + + [$stdout, , $code] = $this->runScript(); + + $this->assertSame(0, $code); + $this->assertSame("current 507\n", $stdout); + $this->assertSame('', $this->upgradeCalls()); + } + + public function testOutdatedSchemaRunsTheSchemaStepThenTheVersionStep(): void + { + $this->seed('1.10.2', '506'); + + [$stdout, , $code] = $this->runScript(); + + $this->assertSame(0, $code); + $this->assertSame("upgraded 506 -> 507\n", $stdout); + $this->assertSame("1 1.10.2 1.10.6 506 507\n3 1.10.2 1.10.6 506 507\n", $this->upgradeCalls()); + } + + public function testSecondRunAfterAnUpgradeIsANoop(): void + { + $this->seed('1.10.2', '506'); + $this->runScript(); + + [$stdout, , $code] = $this->runScript(); + + $this->assertSame(0, $code); + $this->assertSame("current 507\n", $stdout); + $this->assertSame(2, substr_count($this->upgradeCalls(), "\n")); + } + + public function testDieInsideAnUpgradeStepFailsTheRun(): void + { + $this->seed('1.10.2', '506'); + + [$stdout, $stderr, $code] = $this->runScript(['STUB_FAIL_STEP' => '1']); + + $this->assertSame(1, $code); + $this->assertSame('', $stdout); + $this->assertStringContainsString('Unable to update the DB.', $stderr); + } + + public function testDieDuringBootstrapFailsTheRun(): void + { + $this->seed('1.10.2', '506'); + + [$stdout, $stderr, $code] = $this->runScript(['STUB_BOOT_DIE' => '1']); + + $this->assertSame(1, $code); + $this->assertSame('', $stdout); + $this->assertStringContainsString('could not connect to DB', $stderr); + } + + public function testUpgradeThatLeavesTheSchemaOutdatedFails(): void + { + $this->seed('1.10.2', '506'); + + [$stdout, $stderr, $code] = $this->runScript(['STUB_SKIP_SAVE' => '1']); + + $this->assertSame(1, $code); + $this->assertSame('', $stdout); + $this->assertStringContainsString('still needs a database upgrade', $stderr); + } + + public function testPreOneFourSchemaIsRefusedBeforeAnyStep(): void + { + $this->seed('1.3', '100'); + + [, $stderr, $code] = $this->runScript(); + + $this->assertSame(1, $code); + $this->assertStringContainsString('predates YOURLS 1.4', $stderr); + $this->assertSame('', $this->upgradeCalls()); + } + + public function testMissingRootArgumentIsAUsageError(): void + { + [, $stderr, $code] = $this->runScript([], []); + + $this->assertSame(2, $code); + $this->assertStringContainsString('usage:', $stderr); + } +} diff --git a/tests/unit/python/group_vars/test_captcha_enabled.py b/tests/unit/python/group_vars/test_captcha_enabled.py new file mode 100644 index 0000000000..1a1d8e5cc7 --- /dev/null +++ b/tests/unit/python/group_vars/test_captcha_enabled.py @@ -0,0 +1,105 @@ +import unittest +import unittest.mock as mock +from pathlib import Path +from typing import Any, ClassVar + +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar +from ansible.utils.display import Display + +from utils.cache.base import _render_with_templar +from utils.cache.files import PROJECT_ROOT +from utils.cache.yaml import load_yaml_any +from utils.roles.mapping import ROLE_FILE_META_SERVICES + +_GENERAL_VARS = Path(PROJECT_ROOT) / "group_vars" / "all" / "00_general.yml" +_ROLES_DIR = Path(PROJECT_ROOT) / "roles" +_PROVIDERS = {"recaptcha": "RECAPTCHA", "hcaptcha": "HCAPTCHA"} +_CREDENTIALS = {"KEY": "site-key", "SECRET": "site-secret"} + + +def _consumers() -> list[tuple[str, str, str]]: + found = [] + for role_dir in sorted(_ROLES_DIR.iterdir()): + services_file = role_dir / ROLE_FILE_META_SERVICES + if not services_file.is_file(): + continue + services = load_yaml_any(services_file) or {} + for service_key, provider in _PROVIDERS.items(): + entry = services.get(service_key) + enabled = entry.get("enabled") if isinstance(entry, dict) else None + if isinstance(enabled, str) and f"{provider}_ENABLED" in enabled: + found.append((role_dir.name, service_key, enabled)) + return found + + +class TestCaptchaEnabled(unittest.TestCase): + """``services..enabled`` reaches its consumers through the + applications render, whose failure path hands back template text instead + of raising. Inventory ``CAPTCHA`` replaces the group_vars mapping whole, + so a host that configures one provider drops the other's sub-mapping. + """ + + group_vars: ClassVar[dict[str, Any]] + consumers: ClassVar[list[tuple[str, str, str]]] + + @classmethod + def setUpClass(cls): + cls.group_vars = DataLoader().load_from_file( + str(_GENERAL_VARS), trusted_as_template=True + ) + cls.consumers = _consumers() + + def _render(self, value: str, captcha: dict[str, Any]) -> tuple[Any, list[str]]: + variables = {**self.group_vars, "CAPTCHA": captcha} + templar = Templar(loader=DataLoader(), variables=variables) + with mock.patch.object(Display, "deprecated") as deprecated: + rendered = _render_with_templar(value, templar=templar, variables=variables) + coercions = [ + str(call) for call in deprecated.call_args_list if "coerced" in str(call) + ] + return rendered, coercions + + def _assert_every_consumer(self, captcha_for, expected_for): + for role, service_key, value in self.consumers: + provider = _PROVIDERS[service_key] + with self.subTest(role=role, service=service_key): + rendered, coercions = self._render(value, captcha_for(provider)) + self.assertIsInstance(rendered, bool, f"rendered to {rendered!r}") + self.assertEqual(rendered, expected_for(provider)) + self.assertEqual(coercions, []) + + def test_every_provider_has_a_consumer(self): + self.assertEqual( + {service_key for _, service_key, _ in self.consumers}, set(_PROVIDERS) + ) + + def test_the_group_vars_defaults_disable_every_provider(self): + self._assert_every_consumer( + lambda provider: self.group_vars["CAPTCHA"], lambda provider: False + ) + + def test_both_credentials_enable_the_provider(self): + self._assert_every_consumer( + lambda provider: {name: dict(_CREDENTIALS) for name in _PROVIDERS.values()}, + lambda provider: True, + ) + + def test_a_host_that_configures_only_the_other_provider_disables_this_one(self): + def only_the_other(provider): + return { + name: dict(_CREDENTIALS) + for name in _PROVIDERS.values() + if name != provider + } + + self._assert_every_consumer(only_the_other, lambda provider: False) + + def test_a_host_that_configures_only_this_provider_enables_it(self): + self._assert_every_consumer( + lambda provider: {provider: dict(_CREDENTIALS)}, lambda provider: True + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/library/test_directory_batch.py b/tests/unit/python/library/test_directory_batch.py index 2ebc4fcbdc..882690acd7 100644 --- a/tests/unit/python/library/test_directory_batch.py +++ b/tests/unit/python/library/test_directory_batch.py @@ -21,10 +21,13 @@ if TYPE_CHECKING: sys.path.insert(0, str(PROJECT_ROOT)) +from utils import directory_batch + MODULE_PATH = PROJECT_ROOT / "library" / "directory_batch.py" def _load_directory_batch() -> ModuleType: + sys.modules.setdefault("ansible.module_utils.directory_batch", directory_batch) spec = importlib.util.spec_from_file_location( "library_directory_batch", str(MODULE_PATH) ) diff --git a/tests/unit/python/plugins/filter/test_baudolo_seed.py b/tests/unit/python/plugins/filter/test_baudolo_seed.py new file mode 100644 index 0000000000..05b8118c4b --- /dev/null +++ b/tests/unit/python/plugins/filter/test_baudolo_seed.py @@ -0,0 +1,324 @@ +from __future__ import annotations + +import base64 +import io +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +from baudolo.databases import DatabasesCsvError, validate_database +from baudolo.seed.__main__ import check_and_add_entry + +from plugins.filter.baudolo_seed import ( + FilterModule, + _database, + baudolo_seed_apply, + baudolo_seed_current, + baudolo_seeded, +) + +HEADER = "instance;database;username;password\n" + + +def _b64(text: str) -> str: + return base64.b64encode(text.encode("utf-8")).decode("ascii") + + +def _seeded(text: str) -> dict: + return baudolo_seeded(_b64(text)) + + +STABLE = [ + ("central-postgres", "nextcloud", "nextcloud", "pw-1"), + ("central-mariadb", "wordpress", "wordpress", "pa;ss"), + ("central-postgres", "", "postgres", 'quo"te'), + ("central-mariadb", "matomo", "matomo", "multi\nline"), + ("central-postgres", " keycloak ", "keycloak", " spaced "), + ("central-postgres", "gitea", "gitea", ""), + ("central-postgres", "listmonk", "listmonk", "รผnรฏcode"), +] +ENTRIES = [ + *STABLE[:5], + ("central-postgres", "nextcloud", "nextcloud", "pw-2"), + ("central-mariadb", "wordpress", "wordpress", "pa;ss"), + *STABLE[5:], +] + + +def _argv(entry) -> tuple[str, str, str, str]: + instance, name, username, password = entry + return instance, name or "*", username, password + + +def _bytes(path: Path) -> bytes: + if not path.exists(): + return b"" + return path.read_bytes() # nocheck: cache-read baudolo-seed rewrites the tempfile + + +def _b64_of(path: Path) -> str: + return base64.b64encode(_bytes(path)).decode("ascii") + + +def _seed(path: Path, argv) -> None: + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + check_and_add_entry(str(path), *argv) + + +class TestBaudoloSeedEquivalence(unittest.TestCase): + def _per_entry(self, path: Path, entries) -> int: + for entry in entries: + _seed(path, _argv(entry)) + return len(entries) + + def _skipping(self, path: Path, entries) -> int: + state = baudolo_seeded(_b64_of(path)) + calls = 0 + for entry in entries: + argv = _argv(entry) + current = baudolo_seed_current(state, *argv) + state = baudolo_seed_apply(state, *argv) + if not current: + _seed(path, argv) + calls += 1 + return calls + + def _run_both(self, start: bytes | None, entries): + with tempfile.TemporaryDirectory() as tmp: + old = Path(tmp) / "old.csv" + new = Path(tmp) / "new.csv" + if start is not None: + old.write_bytes(start) + new.write_bytes(start) + old_calls = self._per_entry(old, entries) + new_calls = self._skipping(new, entries) + return _bytes(old), _bytes(new), old_calls, new_calls + + def test_fresh_file_matches_and_skips_only_repeats(self): + old, new, old_calls, new_calls = self._run_both(None, ENTRIES) + self.assertEqual(old, new) + self.assertEqual(old_calls, len(ENTRIES)) + self.assertEqual(new_calls, len(ENTRIES) - 1) + + def test_empty_file_from_touch_matches(self): + old, new, _, _ = self._run_both(b"", ENTRIES) + self.assertEqual(old, new) + + def _converged(self, entries) -> bytes: + with tempfile.TemporaryDirectory() as tmp: + first = Path(tmp) / "first.csv" + self._per_entry(first, entries) + return _bytes(first) + + def test_converged_file_needs_no_call_and_keeps_its_bytes(self): + converged = self._converged(STABLE) + old, new, _, new_calls = self._run_both(converged, STABLE) + self.assertEqual(old, new) + self.assertEqual(new, converged) + self.assertEqual(new_calls, 0) + + def test_entries_that_overwrite_each_other_are_seeded_every_time(self): + converged = self._converged(ENTRIES) + old, new, _, new_calls = self._run_both(converged, ENTRIES) + self.assertEqual(old, new) + self.assertEqual(new_calls, 2) + + def test_rotated_password_is_seeded_again(self): + converged = self._converged(STABLE) + rotated = [*STABLE[:-1], ("central-postgres", "listmonk", "listmonk", "rot")] + old, new, _, new_calls = self._run_both(converged, rotated) + self.assertEqual(old, new) + self.assertEqual(new_calls, 1) + + def test_new_consumer_is_appended_after_the_converged_rows(self): + converged = self._converged(STABLE) + grown = [*STABLE, ("central-mariadb", "pixelfed", "pixelfed", "x")] + old, new, _, new_calls = self._run_both(converged, grown) + self.assertEqual(old, new) + self.assertEqual(new_calls, 1) + + def test_invalid_database_name_still_reaches_baudolo(self): + entry = ("central-postgres", "bad name", "u", "p") + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "databases.csv" + self._per_entry(path, STABLE) + with self.assertRaises(DatabasesCsvError): + self._skipping(path, [entry]) + + def test_hand_edited_file_falls_back_to_one_call_per_entry(self): + hand = (HEADER + "central-postgres;nextcloud;nextcloud;pw-2\r\n").encode() + old, new, old_calls, new_calls = self._run_both(hand, ENTRIES) + self.assertEqual(old, new) + self.assertEqual(new_calls, old_calls) + + def test_short_row_falls_back_to_one_call_per_entry(self): + short = (HEADER + "central-postgres;nextcloud\n").encode() + old, new, old_calls, new_calls = self._run_both(short, ENTRIES) + self.assertEqual(old, new) + self.assertEqual(new_calls, old_calls) + + def test_negative_control_a_state_older_than_the_file_breaks_equivalence(self): + converged = self._converged(STABLE) + with tempfile.TemporaryDirectory() as tmp: + stale = Path(tmp) / "stale.csv" + stale.write_bytes(b"") + state = baudolo_seeded(base64.b64encode(converged).decode("ascii")) + for entry in STABLE: + if not baudolo_seed_current(state, *_argv(entry)): + _seed(stale, _argv(entry)) + self.assertNotEqual(_bytes(stale), converged) + + +class TestBaudoloSeeded(unittest.TestCase): + def test_absent_or_empty_file_is_canonical_without_rows(self): + self.assertEqual(baudolo_seeded(""), {"canonical": True, "rows": []}) + + def test_header_only_is_canonical_without_rows(self): + self.assertEqual(_seeded(HEADER), {"canonical": True, "rows": []}) + + def test_rows_are_read_in_file_order(self): + text = HEADER + "a;db1;u1;p1\nb;*;u2;p2\n" + self.assertEqual( + _seeded(text), + { + "canonical": True, + "rows": [["a", "db1", "u1", "p1"], ["b", "*", "u2", "p2"]], + }, + ) + + def test_quoted_fields_stay_canonical(self): + text = HEADER + 'a;db;u;"pa;ss"\nb;db;u;"q""t"\n' + state = _seeded(text) + self.assertTrue(state["canonical"]) + self.assertEqual(state["rows"][0][3], "pa;ss") + self.assertEqual(state["rows"][1][3], 'q"t') + + def test_crlf_is_not_canonical(self): + self.assertFalse(_seeded(HEADER.replace("\n", "\r\n"))["canonical"]) + + def test_missing_trailing_newline_is_not_canonical(self): + self.assertFalse(_seeded(HEADER + "a;db;u;p")["canonical"]) + + def test_blank_line_is_not_canonical(self): + self.assertFalse(_seeded(HEADER + "\na;db;u;p\n")["canonical"]) + + def test_needless_quotes_are_not_canonical(self): + self.assertFalse(_seeded(HEADER + 'a;db;u;"p"\n')["canonical"]) + + def test_foreign_header_is_not_canonical(self): + self.assertFalse(_seeded("instance;database;user;password\n")["canonical"]) + + def test_undecodable_content_is_not_canonical(self): + self.assertFalse(baudolo_seeded("not base64!")["canonical"]) + self.assertFalse( + baudolo_seeded(base64.b64encode(b"\xff\xfe").decode())["canonical"] + ) + + +class TestBaudoloSeedCurrent(unittest.TestCase): + def setUp(self): + self.state = _seeded(HEADER + "a;db;u;p\na;*;su;sp\nb;db;u;p\nb;db;u;q\n") + + def test_same_credentials_are_current(self): + self.assertTrue(baudolo_seed_current(self.state, "a", "db", "u", "p")) + + def test_cluster_row_is_current(self): + self.assertTrue(baudolo_seed_current(self.state, "a", "*", "su", "sp")) + + def test_padded_database_name_matches_the_stripped_row(self): + self.assertTrue(baudolo_seed_current(self.state, "a", " db ", "u", "p")) + + def test_negative_control_other_password_is_not_current(self): + self.assertFalse(baudolo_seed_current(self.state, "a", "db", "u", "p2")) + + def test_negative_control_other_username_is_not_current(self): + self.assertFalse(baudolo_seed_current(self.state, "a", "db", "u2", "p")) + + def test_missing_row_is_not_current(self): + self.assertFalse(baudolo_seed_current(self.state, "c", "db", "u", "p")) + + def test_duplicate_rows_with_other_credentials_are_not_current(self): + self.assertFalse(baudolo_seed_current(self.state, "b", "db", "u", "p")) + + def test_invalid_database_name_is_never_current(self): + state = _seeded(HEADER + "a;bad name;u;p\n") + self.assertFalse(baudolo_seed_current(state, "a", "bad name", "u", "p")) + + def test_non_canonical_state_is_never_current(self): + state = _seeded(HEADER + 'a;db;u;"p"\n') + self.assertFalse(baudolo_seed_current(state, "a", "db", "u", "p")) + + +class TestBaudoloSeedApply(unittest.TestCase): + def test_new_definition_is_appended(self): + state = baudolo_seed_apply(_seeded(HEADER + "a;db;u;p\n"), "b", "*", "u", "p") + self.assertEqual(state["rows"], [["a", "db", "u", "p"], ["b", "*", "u", "p"]]) + + def test_every_matching_row_takes_the_credentials(self): + state = baudolo_seed_apply( + _seeded(HEADER + "a;db;u;p\nx;db;u;p\na;db;v;q\n"), "a", "db", "n", "m" + ) + self.assertEqual( + state["rows"], + [["a", "db", "n", "m"], ["x", "db", "u", "p"], ["a", "db", "n", "m"]], + ) + + def test_database_name_is_stored_stripped(self): + state = baudolo_seed_apply(baudolo_seeded(""), "a", " db ", "u", "p") + self.assertEqual(state["rows"], [["a", "db", "u", "p"]]) + + def test_input_state_is_not_mutated(self): + state = _seeded(HEADER + "a;db;u;p\n") + baudolo_seed_apply(state, "a", "db", "u", "q") + self.assertEqual(state["rows"], [["a", "db", "u", "p"]]) + + def test_non_canonical_state_stays_non_canonical(self): + state = baudolo_seed_apply( + _seeded(HEADER + 'a;db;u;"p"\n'), "a", "db", "u", "p" + ) + self.assertEqual(state, {"canonical": False, "rows": []}) + + def test_invalid_database_name_makes_the_state_non_canonical(self): + state = baudolo_seed_apply(baudolo_seeded(""), "a", "nan", "u", "p") + self.assertFalse(state["canonical"]) + + +class TestDatabaseNameParity(unittest.TestCase): + def test_validation_matches_baudolo(self): + for value in ( + "db", + " db ", + "*", + " * ", + "", + " ", + "nan", + "NaN", + "a-b_c", + "-a", + "a b", + "รผ", + ): + with self.subTest(value=value): + try: + expected = validate_database(value, instance="i") + except DatabasesCsvError: + expected = None + self.assertEqual(_database(value), expected) + + +class TestFilterModule(unittest.TestCase): + def test_filters_are_registered(self): + self.assertEqual( + FilterModule().filters(), + { + "baudolo_seeded": baudolo_seeded, + "baudolo_seed_current": baudolo_seed_current, + "baudolo_seed_apply": baudolo_seed_apply, + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/plugins/lookup/test_docker_cards.py b/tests/unit/python/plugins/lookup/test_docker_cards.py index a450892007..5a981df409 100644 --- a/tests/unit/python/plugins/lookup/test_docker_cards.py +++ b/tests/unit/python/plugins/lookup/test_docker_cards.py @@ -125,6 +125,26 @@ class DummyTlsResolveLookup: return [base_url] +class DummyCanonicalUrlLookup: + """ + Deterministic canonical_url stub. + Emulates: lookup('canonical_url', app_id, key) -> 'http(s)://' + """ + + def run(self, terms, variables=None, **kwargs): + variables = variables or {} + app_id, key = terms + app_domains = variables.get("domains", {}).get(app_id) + if not isinstance(app_domains, dict) or key not in app_domains: + raise AnsibleError( + f"dummy canonical_url: '{app_id}' has no canonical '{key}'" + ) + scheme = ( + "https" if _ansible_bool(variables.get("TLS_ENABLED", True)) else "http" + ) + return [f"{scheme}://{app_domains[key]}"] + + class TestDockerCardsLookup(unittest.TestCase): def setUp(self): self.test_roles_dir = tempfile.mkdtemp(prefix="test_roles_") @@ -174,6 +194,8 @@ logo: return _VarsLookup("domains") if name == "tls": return DummyTlsResolveLookup(templar) + if name == "canonical_url": + return DummyCanonicalUrlLookup() raise AnsibleError(f"Unexpected lookup requested: {name}") docker_cards_module.lookup_loader.get = _patched_get @@ -304,6 +326,39 @@ logo: self.assertEqual(result[0][0]["url"], "http://myportfolio.com") + def _card_target_variables(self, dashboard): + fake_variables = self._base_fake_variables() + fake_variables["domains"] = { + "portfolio": { + "www": "myportfolio.com", + "client": "client.myportfolio.com", + } + } + fake_variables["applications"]["portfolio"]["services"]["dashboard"] = dashboard + return fake_variables + + def test_card_url_targets_the_card_key(self): + fake_variables = self._card_target_variables( + {"enabled": True, "card": "client"} + ) + + result = self._run_lookup(LookupModule(), fake_variables) + + self.assertEqual(result[0][0]["url"], "https://client.myportfolio.com") + + def test_card_url_without_card_key_targets_the_primary_domain(self): + fake_variables = self._card_target_variables({"enabled": True}) + + result = self._run_lookup(LookupModule(), fake_variables) + + self.assertEqual(result[0][0]["url"], "https://myportfolio.com") + + def test_unknown_card_key_fails_loudly(self): + fake_variables = self._card_target_variables({"enabled": True, "card": "nope"}) + + with self.assertRaisesRegex(AnsibleError, "nope"): + self._run_lookup(LookupModule(), fake_variables) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/python/plugins/lookup/test_roles_with_service.py b/tests/unit/python/plugins/lookup/test_roles_with_service.py index 78bc20b99d..d5a3ca45f6 100644 --- a/tests/unit/python/plugins/lookup/test_roles_with_service.py +++ b/tests/unit/python/plugins/lookup/test_roles_with_service.py @@ -72,13 +72,15 @@ class RolesWithServiceLookupTests(unittest.TestCase): handover: frozenset = frozenset(), **kwargs, ): - """Run the lookup with the ``applications``, ``tls`` and - ``network_handover`` sub-lookups patched to stay hermetic. The stubbed - tls answers ``[role_id, "url.base"]`` with the role's own configured - domain, which is what a clearnet deployment resolves; pass ``url_host`` - (every role) or ``url_hosts`` (per role) to model a deployment whose URL - host differs, such as an onion one. A role in ``handover`` hands every - sibling outside its canonical network over.""" + """Run the lookup with the ``applications``, ``tls``, + ``canonical_url`` and ``network_handover`` sub-lookups patched to stay + hermetic; any other sub-lookup raises ``KeyError``. The stubbed tls + answers ``[role_id, "url.base"]`` and the stubbed canonical_url + ``[role_id, key]`` with the role's own configured domain, which is what + a clearnet deployment resolves; pass ``url_host`` (every role) or + ``url_hosts`` (per role) to model a deployment whose URL host differs, + such as an onion one. A role in ``handover`` hands every sibling outside + its canonical network over.""" lookup = self._make_lookup(vars_ or {}) resolve = self.mod._resolve_canonical_domain @@ -100,16 +102,31 @@ class RolesWithServiceLookupTests(unittest.TestCase): role in handover and network_of(domain) != network_of(canonical) ] + named = self.mod._resolve_named_canonical_domain + + class _StubCanonicalUrl: + def run(self, terms_, variables=None, **kwargs): + role, key = str(terms_[0]), str(terms_[1]) + host = ( + (url_hosts or {}).get(role) + or url_host + or named(role, applications.get(role, {}), key) + ) + return [f"https://{host}"] + class _StubApplications: def run(self, terms_, variables=None, **kwargs): return [applications] + stubs = { + "applications": _StubApplications(), + "tls": _StubTls(), + "canonical_url": _StubCanonicalUrl(), + "network_handover": _StubHandover(), + } + def _dispatch(name, **_kwargs): - if name == "applications": - return _StubApplications() - if name == "network_handover": - return _StubHandover() - return _StubTls() + return stubs[name] with patch.object(self.mod.lookup_loader, "get", side_effect=_dispatch): return lookup.run(terms, variables=vars_ or {}, **kwargs) @@ -683,6 +700,60 @@ class RolesWithServiceLookupTests(unittest.TestCase): with self.assertRaises(AnsibleError): self._network_ids({}, network="onion") + def _card_target_role(self, service: str, **block_extra) -> dict: + block = {"enabled": True, "shared": True} + block.update(block_extra) + return { + "web-app-chat": { + "services": {service: block}, + "domains": { + "canonical": { + "synapse": "matrix.example.com", + "element": "element.example.com", + } + }, + }, + } + + def test_dashboard_card_key_targets_that_domain(self): + applications = self._card_target_role("dashboard", card="element") + entry = self._run(["dashboard"], applications)[0][0] + self.assertEqual( + (entry["canonical_domain"], entry["canonical_url"]), + ("element.example.com", "https://element.example.com"), + ) + + def test_dashboard_card_domain_follows_the_onion_url(self): + applications = self._card_target_role("dashboard", card="element") + entry = self._run( + ["dashboard"], applications, url_hosts={"web-app-chat": f"element.{ONION}"} + )[0][0] + self.assertEqual( + (entry["canonical_domain"], entry["canonical_url"]), + (f"element.{ONION}", f"https://element.{ONION}"), + ) + + def test_dashboard_without_card_key_targets_the_primary_domain(self): + applications = self._card_target_role("dashboard") + entry = self._run(["dashboard"], applications)[0][0] + self.assertEqual( + (entry["canonical_domain"], entry["canonical_url"]), + ("matrix.example.com", "https://matrix.example.com"), + ) + + def test_dashboard_unknown_card_key_fails_loudly(self): + applications = self._card_target_role("dashboard", card="nope") + with self.assertRaisesRegex(AnsibleError, "nope"): + self._run(["dashboard"], applications) + + def test_other_services_ignore_a_card_key(self): + applications = self._card_target_role("prometheus", card="element") + entry = self._run(["prometheus"], applications)[0][0] + self.assertEqual( + (entry["canonical_domain"], entry["canonical_url"]), + ("matrix.example.com", "https://matrix.example.com"), + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/python/roles/sys-ctl-hlth-csp/files/test_script.py b/tests/unit/python/roles/sys-ctl-hlth-csp/files/test_script.py index 9b827401f3..0e443f9e70 100644 --- a/tests/unit/python/roles/sys-ctl-hlth-csp/files/test_script.py +++ b/tests/unit/python/roles/sys-ctl-hlth-csp/files/test_script.py @@ -277,6 +277,33 @@ class TestBuildDockerCmdTimeout(unittest.TestCase): ) +class TestBuildDockerCmdConcurrency(unittest.TestCase): + def test_concurrency_arg_precedes_the_url_separator(self) -> None: + cmd = script.build_docker_cmd( + image="img", + urls=["https://a.example/"], + short_mode=True, + ignore_network_blocks_from=["cdn.example"], + concurrency=4, + ) + idx = cmd.index("--concurrency") + self.assertEqual(cmd[idx + 1], "4") + self.assertLess( + idx, + cmd.index("--"), + "an argument after the separator reaches the checker as a URL", + ) + + def test_no_concurrency_arg_without_one(self) -> None: + cmd = script.build_docker_cmd( + image="img", + urls=["https://a.example/"], + short_mode=True, + ignore_network_blocks_from=[], + ) + self.assertNotIn("--concurrency", cmd) + + class TestMainTimesOutOnionsOnly(unittest.TestCase): @patch("script.run_checker") @patch("script.build_urls_from_nginx_confs") diff --git a/tests/unit/python/roles/sys-ctl-hlth-csp/test_unit_flags_are_parsable.py b/tests/unit/python/roles/sys-ctl-hlth-csp/test_unit_flags_are_parsable.py index 953f164ba8..1f66f826c4 100644 --- a/tests/unit/python/roles/sys-ctl-hlth-csp/test_unit_flags_are_parsable.py +++ b/tests/unit/python/roles/sys-ctl-hlth-csp/test_unit_flags_are_parsable.py @@ -21,6 +21,7 @@ _VALUE = { "--tor-confine-allow": "auth.example.org", "--onion-timeout": "1000", "--clearnet-timeout": "1000", + "--concurrency": "4", } _LIST_VALUE = { "--ignore-network-blocks-from": "a.test", @@ -63,6 +64,7 @@ class TestUnitFlagsAreParsable(unittest.TestCase): self.assertIn("--accept-status", found) self.assertIn("--onion-timeout", found) + self.assertIn("--concurrency", found) def test_every_flag_the_unit_passes_is_accepted_by_the_script(self): for flag in sorted(_flags_the_unit_passes()): diff --git a/tests/unit/python/roles/sys-svc-compose/library/__init__.py b/tests/unit/python/roles/sys-svc-compose/library/__init__.py new file mode 100644 index 0000000000..087c384b57 --- /dev/null +++ b/tests/unit/python/roles/sys-svc-compose/library/__init__.py @@ -0,0 +1,5 @@ +from pathlib import Path + +PROJECT_ROOT: Path = Path(__file__).resolve().parents[6] + +__all__ = ["PROJECT_ROOT"] diff --git a/tests/unit/python/roles/sys-svc-compose/library/test_compose_instance_prepare.py b/tests/unit/python/roles/sys-svc-compose/library/test_compose_instance_prepare.py new file mode 100644 index 0000000000..bfcb79fde5 --- /dev/null +++ b/tests/unit/python/roles/sys-svc-compose/library/test_compose_instance_prepare.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +import contextlib +import importlib.util +import io +import json +import os +import stat +import sys +import tempfile +import unittest +from pathlib import Path +from typing import TYPE_CHECKING +from unittest import mock + +from ansible.module_utils.testing import patch_module_args + +from . import PROJECT_ROOT + +if TYPE_CHECKING: + from types import ModuleType + +sys.path.insert(0, str(PROJECT_ROOT)) + +from utils import directory_batch + +MODULE_PATH = ( + PROJECT_ROOT + / "roles" + / "sys-svc-compose" + / "library" + / "compose_instance_prepare.py" +) + +FAKE_COMPOSE = """#!/bin/sh +pwd -P > "{record}" +printf '%s\\n' "$@" >> "{record}" +printf '{stdout}' +printf '{stderr}' >&2 +exit {rc} +""" + + +def _load_module() -> ModuleType: + sys.modules.setdefault("ansible.module_utils.directory_batch", directory_batch) + spec = importlib.util.spec_from_file_location( + "sys_svc_compose_compose_instance_prepare", str(MODULE_PATH) + ) + if spec is None or spec.loader is None: + raise ImportError(f"Could not load: {MODULE_PATH}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) # type: ignore[attr-defined] + return mod + + +_MOD = _load_module() + + +class ComposeInstancePrepareTests(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name).resolve() + self.bin = self.root / "bin" + self.bin.mkdir() + self.record = self.root / "compose.record" + self.instance = self.root / "compose" / "app" + self.marker = self.instance / ".compose-up-pending" + self._fake_compose(stdout="", stderr="", rc=0) + + def tearDown(self): + self._tmp.cleanup() + + def _fake_compose(self, *, stdout, stderr, rc): + script = self.bin / "compose" + script.write_text( + FAKE_COMPOSE.format( + record=self.record, stdout=stdout, stderr=stderr, rc=rc + ), + encoding="utf-8", + ) + script.chmod(0o755) + + def _args(self, **overrides): + args = { + "directories": [str(self.instance), str(self.instance / "volumes")], + "mode": "0755", + "instance": str(self.instance), + "pending_marker": str(self.marker), + "running_check": True, + } + args.update(overrides) + return args + + def _run(self, args, *, check_mode=False, diff=False, path=None): + buffer = io.StringIO() + module_args = {**args, "_ansible_check_mode": check_mode, "_ansible_diff": diff} + with ( + mock.patch.dict(os.environ, {"PATH": str(path or self.bin)}), + patch_module_args(module_args), + contextlib.redirect_stdout(buffer), + self.assertRaises(SystemExit), + ): + _MOD.main() + return json.loads(buffer.getvalue()) + + def _recorded(self): + return self.record.read_text( # nocheck: cache-read tempdir record the fake compose rewrites on every call + encoding="utf-8" + ).splitlines() + + def test_creates_missing_directories_with_the_mode(self): + result = self._run(self._args()) + self.assertTrue(result["changed"]) + self.assertEqual( + result["changed_paths"], + [str(self.instance), str(self.instance / "volumes")], + ) + self.assertEqual(stat.S_IMODE(self.instance.stat().st_mode), 0o755) + + def test_existing_directories_are_unchanged(self): + self._run(self._args()) + result = self._run(self._args()) + self.assertFalse(result["changed"]) + self.assertEqual(result["changed_paths"], []) + + def test_probe_never_reports_changed_on_its_own(self): + self._run(self._args()) + self.marker.write_text("", encoding="utf-8") + result = self._run(self._args()) + self.assertFalse(result["changed"]) + + def test_pending_is_true_when_the_marker_exists(self): + self.instance.mkdir(parents=True) + self.marker.write_text("", encoding="utf-8") + self.assertTrue(self._run(self._args())["pending"]) + + def test_pending_is_false_without_the_marker(self): + self.assertFalse(self._run(self._args())["pending"]) + + def test_pending_counts_a_dangling_symlink_like_stat(self): + self.instance.mkdir(parents=True) + self.marker.symlink_to(self.root / "gone") + self.assertTrue(self._run(self._args())["pending"]) + + def test_running_reports_compose_ps_output_without_trailing_newlines(self): + self._fake_compose(stdout="abc\\ndef\\n", stderr="warn\\n", rc=0) + running = self._run(self._args())["running"] + self.assertEqual(running, {"rc": 0, "stdout": "abc\ndef", "stderr": "warn"}) + + def test_running_runs_ps_with_the_running_filter_in_the_instance(self): + self._run(self._args()) + self.assertEqual( + self._recorded(), + [str(self.instance), "ps", "-q", "--filter", "status=running"], + ) + + def test_running_reports_a_failing_ps_without_failing_the_module(self): + self._fake_compose(stdout="", stderr="no configuration file provided", rc=1) + result = self._run(self._args()) + self.assertNotIn("failed", result) + self.assertEqual(result["running"]["rc"], 1) + self.assertEqual(result["running"]["stderr"], "no configuration file provided") + + def test_running_is_null_and_ps_skipped_without_running_check(self): + result = self._run(self._args(running_check=False)) + self.assertIsNone(result["running"]) + self.assertFalse(self.record.exists()) + + def test_pending_is_reported_without_running_check(self): + self.instance.mkdir(parents=True) + self.marker.write_text("", encoding="utf-8") + self.assertTrue(self._run(self._args(running_check=False))["pending"]) + + def test_check_mode_creates_nothing_and_skips_ps(self): + result = self._run(self._args(), check_mode=True) + self.assertTrue(result["changed"]) + self.assertIsNone(result["running"]) + self.assertFalse(self.instance.exists()) + self.assertFalse(self.record.exists()) + + def test_directory_failure_fails_before_ps(self): + self.instance.parent.mkdir(parents=True) + self.instance.write_text("", encoding="utf-8") + result = self._run(self._args()) + self.assertTrue(result["failed"]) + self.assertIn("already exists as a file", result["msg"]) + self.assertFalse(self.record.exists()) + + def test_missing_compose_fails_loudly(self): + empty = self.root / "empty" + empty.mkdir() + result = self._run(self._args(), path=empty) + self.assertTrue(result["failed"]) + self.assertIn("compose", result["msg"]) + + def test_missing_compose_is_irrelevant_without_running_check(self): + empty = self.root / "empty" + empty.mkdir() + result = self._run(self._args(running_check=False), path=empty) + self.assertNotIn("failed", result) + + def test_diff_mode_lists_each_created_directory(self): + result = self._run(self._args(), diff=True) + self.assertEqual(len(result["diff"]), 2) + self.assertEqual(result["diff"][0]["after"]["state"], "directory") + + def test_running_check_is_required(self): + args = self._args() + del args["running_check"] + result = self._run(args) + self.assertTrue(result["failed"]) + self.assertIn("running_check", result["msg"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/roles/web-app-discourse/test_legacy_checkout_order.py b/tests/unit/python/roles/web-app-discourse/test_legacy_checkout_order.py new file mode 100644 index 0000000000..b0937cd1d3 --- /dev/null +++ b/tests/unit/python/roles/web-app-discourse/test_legacy_checkout_order.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import unittest + +from utils.cache.files import PROJECT_ROOT, read_text +from utils.cache.yaml import load_yaml_str + +LAUNCHER = PROJECT_ROOT / "roles/web-app-discourse/tasks/03_launcher.yml" +CLEANUP = "launcher/legacy_checkout.yml" +GIT_MODULES = {"git", "ansible.builtin.git"} + + +def _included_file(task: dict) -> str | None: + include = task.get("include_tasks") + return include.get("file") if isinstance(include, dict) else include + + +class TestDiscourseLegacyCheckoutOrder(unittest.TestCase): + def test_the_url_named_checkout_is_retired_before_the_clone(self) -> None: + tasks = load_yaml_str(read_text(str(LAUNCHER))) + cleanup = next( + index for index, task in enumerate(tasks) if _included_file(task) == CLEANUP + ) + clone = next( + index for index, task in enumerate(tasks) if GIT_MODULES & task.keys() + ) + self.assertLess( + cleanup, + clone, + "the git module clones from the services directory, where git takes " + "https://github.com/... for the old checkout's relative path " + "https:/github.com/... and aborts with 'protocol ... is not supported' " + "while that checkout exists", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/roles/web-app-nextcloud/filter_plugins/test_nextcloud_plugin_config.py b/tests/unit/python/roles/web-app-nextcloud/filter_plugins/test_nextcloud_plugin_config.py new file mode 100644 index 0000000000..e57e10259f --- /dev/null +++ b/tests/unit/python/roles/web-app-nextcloud/filter_plugins/test_nextcloud_plugin_config.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import importlib.util +import sys +import unittest + +from ansible.template import Templar, trust_as_template + +from . import PROJECT_ROOT + + +def _load_module(rel_path: str, name: str): + path = PROJECT_ROOT / rel_path + spec = importlib.util.spec_from_file_location(name, str(path)) + mod = importlib.util.module_from_spec(spec) + assert spec and spec.loader + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + +config_mod = _load_module( + "roles/web-app-nextcloud/filter_plugins/nextcloud_plugin_config.py", + "nextcloud_plugin_config", +) +nextcloud_plugin_config_pending = config_mod.nextcloud_plugin_config_pending +nextcloud_occ_value = config_mod.nextcloud_occ_value + +PREVIOUS_VALUE_EXPRESSION = ( + "{{ ( (item.configvalue | to_json) if (item.configvalue is mapping or " + "(item.configvalue is sequence and item.configvalue is not string)) " + "else (item.configvalue | string) ) }}" +) + + +def _spec(*entries): + return {"config": {"plugin_configuration": list(entries)}} + + +def _entry(appid, configkey, configvalue, **extra): + return {"appid": appid, "configkey": configkey, "configvalue": configvalue, **extra} + + +def _apps(**apps): + return {"system": {}, "apps": apps} + + +def _keys(pending): + return [(item["appid"], item["configkey"]) for item in pending] + + +class TestNextcloudOccValue(unittest.TestCase): + def test_matches_the_previous_jinja_rendering(self): + values = [ + 43200, + 0, + "yes", + "", + "groupOfUniqueNames\nposixGroup", + "it's", + None, + True, + 1.5, + [{"name": "XWiki", "url": "https://wiki.example/"}], + {"custom_oidc": [{"name": "kc", "groupMapping": {"/a": "admin"}}]}, + {"รผmlaut": "รฉ", "nested": [1, 2.0, None, False]}, + '{"servers": []}', + ] + for value in values: + with self.subTest(value=value): + rendered = Templar(variables={"item": {"configvalue": value}}).template( + trust_as_template("=" + PREVIOUS_VALUE_EXPRESSION) + ) + self.assertEqual("=" + nextcloud_occ_value(value), rendered) + + +class TestNextcloudPluginConfigPending(unittest.TestCase): + def test_key_nextcloud_lacks_is_pending(self): + pending = nextcloud_plugin_config_pending( + [_spec(_entry("user_ldap", "background_sync_interval", 43200))], + _apps(user_ldap={"enabled": "yes"}), + ) + self.assertEqual( + pending, + [ + { + "appid": "user_ldap", + "configkey": "background_sync_interval", + "type": None, + "value": "43200", + } + ], + ) + + def test_held_string_value_is_dropped(self): + pending = nextcloud_plugin_config_pending( + [_spec(_entry("user_ldap", "s01ldap_host", "ldap"))], + _apps(user_ldap={"s01ldap_host": "ldap"}), + ) + self.assertEqual(pending, []) + + def test_held_integer_value_is_dropped(self): + pending = nextcloud_plugin_config_pending( + [_spec(_entry("user_ldap", "s01ldap_port", 389))], + _apps(user_ldap={"s01ldap_port": 389}), + ) + self.assertEqual(pending, []) + + def test_negative_control_near_miss_values_stay_pending(self): + pending = nextcloud_plugin_config_pending( + [ + _spec( + _entry("a", "case", "Value"), + _entry("a", "space", "v"), + _entry("a", "number", 389), + _entry("a", "padded", "7"), + ) + ], + _apps(a={"case": "value", "space": "v ", "number": 390, "padded": 7.0}), + ) + self.assertEqual( + _keys(pending), + [("a", "case"), ("a", "space"), ("a", "number"), ("a", "padded")], + ) + + def test_boolean_value_is_always_pending(self): + pending = nextcloud_plugin_config_pending( + [_spec(_entry("a", "flag", "1"), _entry("a", "other", "True"))], + _apps(a={"flag": True, "other": True}), + ) + self.assertEqual(_keys(pending), [("a", "flag"), ("a", "other")]) + + def test_masked_sensitive_value_is_always_pending(self): + masked = config_mod.SENSITIVE_VALUE + pending = nextcloud_plugin_config_pending( + [_spec(_entry("onlyoffice", "jwt_secret", masked))], + _apps(onlyoffice={"jwt_secret": masked}), + ) + self.assertEqual(_keys(pending), [("onlyoffice", "jwt_secret")]) + + def test_typed_entry_is_always_pending_with_its_type(self): + pending = nextcloud_plugin_config_pending( + [_spec(_entry("spreed", "stun_servers", "[]", type="array"))], + _apps(spreed={"stun_servers": "[]"}), + ) + self.assertEqual(pending[0]["type"], "array") + self.assertEqual(pending[0]["value"], "[]") + + def test_structured_value_held_as_its_json_string_is_dropped(self): + value = [{"name": "XWiki", "url": "https://wiki.example/"}] + pending = nextcloud_plugin_config_pending( + [_spec(_entry("xwiki", "instances", value))], + _apps(xwiki={"instances": nextcloud_occ_value(value)}), + ) + self.assertEqual(pending, []) + + def test_structured_value_held_decoded_is_pending(self): + value = [{"name": "XWiki"}] + pending = nextcloud_plugin_config_pending( + [_spec(_entry("xwiki", "instances", value))], + _apps(xwiki={"instances": value}), + ) + self.assertEqual(_keys(pending), [("xwiki", "instances")]) + + def test_order_follows_specs_and_entries(self): + pending = nextcloud_plugin_config_pending( + [ + _spec( + _entry("onlyoffice", "jwt_secret", "s"), + _entry("onlyoffice", "DocumentServerUrl", "u"), + ), + {"enabled": True}, + _spec(_entry("bbb", "api.url", "b")), + ], + _apps(), + ) + self.assertEqual( + _keys(pending), + [ + ("onlyoffice", "jwt_secret"), + ("onlyoffice", "DocumentServerUrl"), + ("bbb", "api.url"), + ], + ) + + def test_spec_without_configuration_adds_nothing(self): + pending = nextcloud_plugin_config_pending( + [{"enabled": True}, {"config": None}, {"config": {"archive": "x"}}], + _apps(), + ) + self.assertEqual(pending, []) + + def test_empty_php_arrays_count_as_no_configuration(self): + pending = nextcloud_plugin_config_pending( + [_spec(_entry("deck", "k", "v"), _entry("news", "k", "v"))], + {"system": [], "apps": {"deck": []}}, + ) + self.assertEqual(_keys(pending), [("deck", "k"), ("news", "k")]) + pending = nextcloud_plugin_config_pending( + [_spec(_entry("deck", "k", "v"))], {"system": [], "apps": []} + ) + self.assertEqual(_keys(pending), [("deck", "k")]) + + +class TestFilterModule(unittest.TestCase): + def test_filter_is_registered(self): + self.assertEqual( + config_mod.FilterModule().filters(), + {"nextcloud_plugin_config_pending": nextcloud_plugin_config_pending}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/roles/web-app-nextcloud/filter_plugins/test_nextcloud_upgrade.py b/tests/unit/python/roles/web-app-nextcloud/filter_plugins/test_nextcloud_upgrade.py index ced0016ffe..90fff94447 100644 --- a/tests/unit/python/roles/web-app-nextcloud/filter_plugins/test_nextcloud_upgrade.py +++ b/tests/unit/python/roles/web-app-nextcloud/filter_plugins/test_nextcloud_upgrade.py @@ -210,5 +210,198 @@ class TestNextcloudStatus(unittest.TestCase): MODULE.nextcloud_status("Nextcloud is not installed") +class TestNextcloudNeedsPostUpgrade(unittest.TestCase): + def test_unchanged_redeploy_skips_the_pass(self): + self.assertFalse( + MODULE.nextcloud_needs_post_upgrade(STATUS_JSON, "32.0.1.2", "32.0.1.2") + ) + + def test_minor_upgrade_in_this_deploy_runs_the_pass(self): + self.assertTrue( + MODULE.nextcloud_needs_post_upgrade(STATUS_JSON, "32.0.0.13", "32.0.0.13") + ) + + def test_major_upgrade_in_this_deploy_runs_the_pass(self): + self.assertTrue( + MODULE.nextcloud_needs_post_upgrade(STATUS_JSON, "31.0.11.1", "31.0.11.1") + ) + + def test_fresh_install_runs_the_pass(self): + self.assertTrue(MODULE.nextcloud_needs_post_upgrade(STATUS_JSON, "", "")) + + def test_deploy_that_stopped_before_the_repair_runs_the_pass_again(self): + self.assertTrue( + MODULE.nextcloud_needs_post_upgrade(STATUS_JSON, "32.0.1.2", "32.0.0.13") + ) + + def test_version_change_runs_the_pass_even_when_the_marker_already_matches(self): + self.assertTrue( + MODULE.nextcloud_needs_post_upgrade(STATUS_JSON, "32.0.0.13", "32.0.1.2") + ) + + def test_missing_local_data_volume_falls_back_to_the_marker(self): + self.assertFalse( + MODULE.nextcloud_needs_post_upgrade(STATUS_JSON, "", "32.0.1.2") + ) + + def test_php_notice_before_the_marker_is_skipped(self): + output = "PHP Deprecated: something in /var/www/html/lib/x.php\n32.0.1.2\n" + self.assertFalse( + MODULE.nextcloud_needs_post_upgrade(STATUS_JSON, "32.0.1.2", output) + ) + + def test_php_notice_without_a_marker_runs_the_pass(self): + output = "PHP Deprecated: something in /var/www/html/lib/x.php\n\n" + self.assertTrue( + MODULE.nextcloud_needs_post_upgrade(STATUS_JSON, "32.0.1.2", output) + ) + + def test_null_marker_runs_the_pass(self): + self.assertTrue( + MODULE.nextcloud_needs_post_upgrade(STATUS_JSON, "32.0.1.2", "null") + ) + + def test_status_without_version_is_rejected(self): + status = ( + '{"installed":true,"versionstring":"32.0.1","maintenance":false,' + '"needsDbUpgrade":false}' + ) + with self.assertRaises(AnsibleFilterError): + MODULE.nextcloud_needs_post_upgrade(status, "32.0.1.2", "32.0.1.2") + + +class TestPhpVersionCompare(unittest.TestCase): + def test_newer_patch_is_greater(self): + self.assertEqual(MODULE._php_version_compare("1.17.0", "1.16.2"), 1) + + def test_identical_versions_are_equal(self): + self.assertEqual(MODULE._php_version_compare("10.2.1", "10.2.1"), 0) + + def test_numeric_parts_compare_as_numbers(self): + self.assertEqual(MODULE._php_version_compare("1.10.0", "1.9.9"), 1) + + def test_shorter_version_is_older(self): + self.assertEqual(MODULE._php_version_compare("5.2", "5.2.0"), -1) + + def test_release_is_newer_than_its_beta(self): + self.assertEqual(MODULE._php_version_compare("2.0.0", "2.0.0-beta.1"), 1) + + def test_release_candidate_is_older_than_the_release(self): + self.assertEqual(MODULE._php_version_compare("1.0rc1", "1.0"), -1) + + def test_alpha_is_older_than_beta(self): + self.assertEqual(MODULE._php_version_compare("1.0.0-alpha", "1.0.0-beta"), -1) + + def test_patch_level_is_newer_than_the_release(self): + self.assertEqual(MODULE._php_version_compare("1.0pl1", "1.0"), 1) + + def test_empty_version_is_older(self): + self.assertEqual(MODULE._php_version_compare("", "1.0"), -1) + + +def _update_check(found): + return '{"apps":{"updatenotification":' + found + "}}" + + +INSTALLED_APPS_JSON = ( + '{"enabled":{"deck":"1.16.2","onlyoffice":"10.2.1","files":"2.4.0",' + '"calendar":"6.0.1"}}' +) + + +class TestNextcloudCachedAppUpdates(unittest.TestCase): + def test_newer_release_of_an_unpinned_app_is_updated(self): + output = _update_check('{"deck":"1.17.0","enabled":"yes"}') + self.assertEqual( + MODULE.nextcloud_cached_app_updates( + output, INSTALLED_APPS_JSON, ["onlyoffice"] + ), + ["deck"], + ) + + def test_newer_release_of_a_pinned_app_is_left_alone(self): + output = _update_check('{"onlyoffice":"10.3.0"}') + self.assertEqual( + MODULE.nextcloud_cached_app_updates( + output, INSTALLED_APPS_JSON, ["onlyoffice"] + ), + [], + ) + + def test_release_already_installed_is_not_updated(self): + output = _update_check('{"deck":"1.16.2"}') + self.assertEqual( + MODULE.nextcloud_cached_app_updates(output, INSTALLED_APPS_JSON, []), [] + ) + + def test_release_older_than_the_installed_one_is_not_updated(self): + output = _update_check('{"calendar":"5.5.0"}') + self.assertEqual( + MODULE.nextcloud_cached_app_updates(output, INSTALLED_APPS_JSON, []), [] + ) + + def test_empty_update_check_means_no_update(self): + self.assertEqual( + MODULE.nextcloud_cached_app_updates( + _update_check("[]"), INSTALLED_APPS_JSON, [] + ), + [], + ) + + def test_update_check_without_its_app_entry_means_no_update(self): + self.assertEqual( + MODULE.nextcloud_cached_app_updates('{"apps":[]}', INSTALLED_APPS_JSON, []), + [], + ) + + def test_own_settings_of_the_update_check_are_not_apps(self): + output = _update_check( + '{"core":"34.0.6.1","enabled":"yes","installed_version":"1.22.0",' + '"update_check_errors":0}' + ) + self.assertEqual( + MODULE.nextcloud_cached_app_updates(output, INSTALLED_APPS_JSON, []), [] + ) + + def test_release_of_an_app_that_is_not_enabled_is_ignored(self): + output = _update_check('{"news":"27.0.0"}') + self.assertEqual( + MODULE.nextcloud_cached_app_updates(output, INSTALLED_APPS_JSON, []), [] + ) + + def test_app_without_an_installed_version_is_skipped(self): + output = _update_check('{"deck":"1.17.0"}') + self.assertEqual( + MODULE.nextcloud_cached_app_updates( + output, '{"enabled":{"deck":true}}', [] + ), + [], + ) + + def test_empty_cached_release_means_no_update(self): + output = _update_check('{"deck":""}') + self.assertEqual( + MODULE.nextcloud_cached_app_updates(output, INSTALLED_APPS_JSON, []), [] + ) + + def test_several_updates_are_sorted(self): + output = _update_check('{"deck":"1.17.0","calendar":"6.1.0"}') + self.assertEqual( + MODULE.nextcloud_cached_app_updates(output, INSTALLED_APPS_JSON, []), + ["calendar", "deck"], + ) + + def test_php_notice_before_the_json_is_skipped(self): + output = "PHP Deprecated: x in y.php\n" + _update_check('{"deck":"1.17.0"}') + self.assertEqual( + MODULE.nextcloud_cached_app_updates(output, INSTALLED_APPS_JSON, []), + ["deck"], + ) + + def test_update_check_output_without_json_is_rejected(self): + with self.assertRaises(AnsibleFilterError): + MODULE.nextcloud_cached_app_updates("broken", INSTALLED_APPS_JSON, []) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/python/utils/roles/applications/test_dashboard.py b/tests/unit/python/utils/roles/applications/test_dashboard.py new file mode 100644 index 0000000000..77a6b601a0 --- /dev/null +++ b/tests/unit/python/utils/roles/applications/test_dashboard.py @@ -0,0 +1,46 @@ +"""Unit tests for ``utils/roles/applications/dashboard.py``.""" + +from __future__ import annotations + +import unittest + +from ansible.errors import AnsibleError + +from utils.roles.applications.dashboard import card_target_key + + +class TestCardTargetKey(unittest.TestCase): + def test_a_binding_without_the_key_targets_the_primary_domain(self): + self.assertEqual( + "", card_target_key("web-app-foo", {"enabled": True, "shared": True}) + ) + + def test_a_missing_binding_targets_the_primary_domain(self): + self.assertEqual("", card_target_key("web-app-foo", None)) + + def test_the_named_key_is_returned(self): + self.assertEqual( + "element", + card_target_key("web-app-matrix", {"card": "element"}), + ) + + def test_surrounding_whitespace_is_dropped(self): + self.assertEqual( + "webmail", + card_target_key("web-app-stalwart", {"card": " webmail "}), + ) + + def test_an_alias_canonical_key_is_not_a_card_target(self): + self.assertEqual("", card_target_key("web-svc-cdn", {"canonical": "cdn"})) + + def test_an_empty_key_fails_loudly(self): + with self.assertRaisesRegex(AnsibleError, "web-app-foo"): + card_target_key("web-app-foo", {"card": " "}) + + def test_a_non_string_key_fails_loudly(self): + with self.assertRaises(AnsibleError): + card_target_key("web-app-foo", {"card": ["element"]}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/test_directory_batch.py b/tests/unit/python/utils/test_directory_batch.py new file mode 100644 index 0000000000..82a924a203 --- /dev/null +++ b/tests/unit/python/utils/test_directory_batch.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.testing import patch_module_args + +from utils.directory_batch import ensure_all + + +def _entry(path, mode="0755"): + return {"path": str(path), "mode": mode, "owner": None, "group": None} + + +class EnsureAllTests(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + + def tearDown(self): + self._tmp.cleanup() + + def _ensure(self, entries, *, check_mode=False): + with patch_module_args({"_ansible_check_mode": check_mode}): + module = AnsibleModule(argument_spec={}, supports_check_mode=True) + return ensure_all(module, entries) + + def test_returns_the_changed_paths_in_order_with_one_diff_each(self): + first = self.root / "a" + second = self.root / "b" + changed_paths, diffs = self._ensure([_entry(first), _entry(second)]) + self.assertEqual(changed_paths, [str(first), str(second)]) + self.assertEqual( + [diff["after"]["path"] for diff in diffs], [str(first), str(second)] + ) + + def test_unchanged_paths_are_left_out(self): + existing = self.root / "kept" + existing.mkdir() + existing.chmod(0o755) + self.assertEqual(self._ensure([_entry(existing)]), ([], [])) + + def test_empty_entries_change_nothing(self): + self.assertEqual(self._ensure([]), ([], [])) + + def test_check_mode_reports_without_creating(self): + target = self.root / "new" + changed_paths, _ = self._ensure([_entry(target)], check_mode=True) + self.assertEqual(changed_paths, [str(target)]) + self.assertFalse(target.exists()) + + def test_first_error_fails_with_the_paths_changed_before_it(self): + created = self.root / "created" + blocker = self.root / "blocker" + never = self.root / "never" + blocker.write_text("", encoding="utf-8") + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer), self.assertRaises(SystemExit): + self._ensure([_entry(created), _entry(blocker), _entry(never)]) + result = json.loads(buffer.getvalue()) + self.assertTrue(result["failed"]) + self.assertEqual(result["path"], str(blocker)) + self.assertEqual(result["changed_paths"], [str(created)]) + self.assertFalse(never.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/directory_batch.py b/utils/directory_batch.py new file mode 100644 index 0000000000..25506cdc1d --- /dev/null +++ b/utils/directory_batch.py @@ -0,0 +1,101 @@ +import os +from pathlib import Path + +ATTRIBUTES = ("mode", "owner", "group") + +ATTRIBUTE_SPEC = { + "mode": {"type": "raw"}, + "owner": {"type": "str"}, + "group": {"type": "str"}, +} + + +class DirectoryError(Exception): + pass + + +def path_state(path): + target = Path(path) + if target.is_symlink(): + return "link" + if target.is_dir(): + return "directory" + if target.exists(): + return "file" + return "absent" + + +def apply_attributes(module, path, entry, changed, diff): + file_args = module.load_file_common_arguments( + {"path": path, **{name: entry[name] for name in ATTRIBUTES}} + ) + return module.set_fs_attributes_if_different(file_args, changed, diff, expand=False) + + +def create(module, path, entry, diff): + changed = False + current = "" + for part in path.strip("/").split("/"): + current = f"{current}/{part}" + candidate = Path(current) + if candidate.exists(): + continue + try: + candidate.mkdir() + changed = True + except FileExistsError: + if not candidate.is_dir(): + raise + changed = apply_attributes(module, current, entry, changed, diff) + return changed + + +def ensure(module, entry): + path = entry["path"] + if not Path(path).is_absolute(): + raise DirectoryError(f"{path!r} is not an absolute path") + if Path(path).is_symlink(): + path = os.path.realpath(path) + state = path_state(path) + diff = {"before": {"path": path}, "after": {"path": path}} + if state == "directory": + return apply_attributes(module, path, entry, False, diff), diff + if state != "absent": + raise DirectoryError(f"{path} already exists as a {state}") + diff["before"]["state"] = state + diff["after"]["state"] = "directory" + if module.check_mode: + return True, diff + try: + return create(module, path, entry, diff), diff + except OSError as error: + raise DirectoryError(f"Could not create {path}: {error}") from error + + +def ensure_all(module, entries): + """Ensure every entry in order and fail the module on the first error. + + Args: + module: the running ``AnsibleModule``. + entries: list of ``{path, mode, owner, group}``. + + Returns: + ``(changed_paths, diffs)``: the paths that were created or whose + attributes changed, and one ``before``/``after`` record per such path. + """ + changed_paths = [] + diffs = [] + for entry in entries: + try: + changed, diff = ensure(module, entry) + except DirectoryError as error: + module.fail_json( + msg=str(error), + path=entry["path"], + changed=bool(changed_paths), + changed_paths=changed_paths, + ) + if changed: + changed_paths.append(entry["path"]) + diffs.append(diff) + return changed_paths, diffs diff --git a/utils/roles/applications/dashboard.py b/utils/roles/applications/dashboard.py new file mode 100644 index 0000000000..403df776e5 --- /dev/null +++ b/utils/roles/applications/dashboard.py @@ -0,0 +1,42 @@ +"""Target of an application's dashboard card. + +A role's ``services.dashboard`` binding MAY name ``card: ``, a key of its +``domains.canonical`` mapping. The card, the Apps menu entry and the dashboard +tile-reachability target then point at that domain instead of the role's +primary domain. +""" + +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleError + +DASHBOARD_SERVICE = "dashboard" +CARD_TARGET_KEY = "card" + + +def card_target_key(application_id: str, dashboard: Any) -> str: + """Return the ``domains.canonical`` key the dashboard card targets. + + Args: + application_id: the role whose binding is read, named in the error. + dashboard: the role's ``services.dashboard`` binding. + + Returns: + The named key, or ``""`` when the binding names none and the card + targets the primary domain. + + Raises: + AnsibleError: the binding names something other than a non-empty + string. + """ + if not isinstance(dashboard, dict) or CARD_TARGET_KEY not in dashboard: + return "" + key = dashboard[CARD_TARGET_KEY] + if not isinstance(key, str) or not key.strip(): + raise AnsibleError( + f"{application_id}: services.{DASHBOARD_SERVICE}.{CARD_TARGET_KEY} " + f"must name a domains.canonical key, got {key!r}" + ) + return key.strip()