diff --git a/.dockerignore b/.dockerignore index d2098ebad7..426c33f6c1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,6 +14,7 @@ venv *tree.json roles/list.json roles/importer_result.json +roles/web-app-cst-* *.pyc *.egg-info build/* diff --git a/.gitignore b/.gitignore index d2098ebad7..426c33f6c1 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ venv *tree.json roles/list.json roles/importer_result.json +roles/web-app-cst-* *.pyc *.egg-info build/* diff --git a/Makefile b/Makefile index eb19a6e74b..ee1bcc5f68 100644 --- a/Makefile +++ b/Makefile @@ -638,6 +638,20 @@ requirements-archive: @"$${PYTHON}" -m pip install --quiet --upgrade kpmx @"$${PYTHON}" -m pkgmgr archive docs/requirements +.PHONY: roles-clean +# Remove every custom role (roles/web-app-cst-*) a pull placed in this checkout and nothing else. +roles-clean: + @"$${PYTHON}" -m cli.administration.roles.clean + +.PHONY: roles-pull +# Pull the custom roles (web-app-cst-*) an inventory declares into roles/ without deploying, removing the ones it no longer declares. +# Usage: make roles-pull inventory= +# Example: make roles-pull inventory=inventories/production/devices.yml +# Param inventory: inventory file whose group_vars/host_vars declare applications..custom and .source (required). +roles-pull: + @test -n '$(inventory)' || { echo 'usage: make roles-pull inventory='; exit 2; } + @"$${PYTHON}" -m cli.administration.roles.pull '$(inventory)' + .PHONY: roundtrip # Validate one or more roles through every deploy mode in order (compose, then swarm), stopping at the first failure. # Param apps: space-separated role ids; default = one role per base cluster, most-complex first (complexity --unique). diff --git a/aliases b/aliases index 00d47d37a3..f771a25946 100644 --- a/aliases +++ b/aliases @@ -125,6 +125,8 @@ alias i8rece='m readme-check' alias i8rege='m readme-generate' alias i8rein='m readme-index' alias i8ro='m roundtrip' +alias i8rocl='m roles-clean' +alias i8ropu='m roles-pull' alias i8ru='m runner-ci-deploy' alias i8ruci='m runner-ci-disable' alias i8rucine='m runner-ci-enable' diff --git a/cli/administration/deploy/container/command.py b/cli/administration/deploy/container/command.py index 99d4131fa3..5d1e2c0865 100644 --- a/cli/administration/deploy/container/command.py +++ b/cli/administration/deploy/container/command.py @@ -266,6 +266,7 @@ def run_in_container( "-vv", "--assert", "true", + "--skip-pull", *deploy_args, ] print(">>> Deploy command inside container:") diff --git a/cli/administration/deploy/dedicated/command.py b/cli/administration/deploy/dedicated/command.py index 3e2a4cfdee..24adfba58e 100644 --- a/cli/administration/deploy/dedicated/command.py +++ b/cli/administration/deploy/dedicated/command.py @@ -4,6 +4,8 @@ import argparse import subprocess from typing import Any +from cli.administration.roles.pull import pull_or_exit + from .apps import validate_application_ids from .modes import add_dynamic_mode_args, build_modes_from_args, load_modes_from_yaml from .paths import INVENTORY_VALIDATOR_PATH, MODES_FILE, PLAYBOOK_PATH, PROJECT_ROOT @@ -117,6 +119,14 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "-B", "--skip-build", action="store_true", help="Skip build phase." ) + parser.add_argument( + "--skip-pull", + action="store_true", + help=( + "Deploy the custom roles already under roles/ instead of pulling " + "the ones the inventory declares (generated test inventories)." + ), + ) parser.add_argument( "--id", nargs="+", @@ -185,6 +195,16 @@ def parse_args( return args, passthrough, modes_spec +def pull_custom_roles(args: Any) -> None: + """Place the custom roles the inventory declares before anything reads roles/. + + Args: + args: parsed wrapper args; ``skip_pull`` keeps roles/ as it is. + """ + if not args.skip_pull: + pull_or_exit(args.inventory) + + def run_from_args( args: Any, passthrough: list[str], @@ -221,4 +241,5 @@ def main(argv: list[str] | None = None) -> int: ``cli.administration.deploy.swarm``. """ args, passthrough, modes_spec = parse_args(argv) + pull_custom_roles(args) return run_from_args(args, passthrough, modes_spec) diff --git a/cli/administration/deploy/development/deploy/run.py b/cli/administration/deploy/development/deploy/run.py index 57d2a82a32..c5dfa43a20 100644 --- a/cli/administration/deploy/development/deploy/run.py +++ b/cli/administration/deploy/development/deploy/run.py @@ -51,6 +51,7 @@ def _run_deploy( "--assert", "true", "--diff", + "--skip-pull", "--id", *deploy_ids, ] diff --git a/cli/administration/deploy/development/env.py b/cli/administration/deploy/development/env.py index c5c705cb44..3bfb98f077 100644 --- a/cli/administration/deploy/development/env.py +++ b/cli/administration/deploy/development/env.py @@ -15,6 +15,7 @@ def compose_file_args() -> list[str]: stack outright, so a checkout that was never handed a shared git dir or cache network simply does not layer that file in. """ + from .extra_mounts import parse_mounts, write_mounts_override from .profile import Profile profile = Profile() @@ -33,6 +34,11 @@ def compose_file_args() -> list[str]: from .runner_mirror import write_override out += ["-f", write_override(PROJECT_ROOT, out[1::2])] + mounts = parse_mounts(os.environ.get("INFINITO_EXTRA_MOUNTS") or "") + if mounts: + from utils.cache.files import PROJECT_ROOT + + out += ["-f", write_mounts_override(PROJECT_ROOT, mounts)] return out diff --git a/cli/administration/deploy/development/extra_mounts.py b/cli/administration/deploy/development/extra_mounts.py new file mode 100644 index 0000000000..794dabdc2e --- /dev/null +++ b/cli/administration/deploy/development/extra_mounts.py @@ -0,0 +1,47 @@ +"""Mount extra host directories read-only into the infinito container.""" + +from __future__ import annotations + +from pathlib import Path + +from utils.cache.yaml import dump_yaml + +OVERRIDE = "build/compose/extra-mounts.override.yml" + + +def parse_mounts(raw: str) -> list[Path]: + """Return the directories of a comma-separated ``INFINITO_EXTRA_MOUNTS`` value. + + Args: + raw: comma-separated absolute host directories. + + Raises: + SystemExit: an entry is not an existing absolute directory. + """ + directories: list[Path] = [] + for item in filter(None, (part.strip() for part in raw.split(","))): + directory = Path(item) + if not directory.is_absolute() or not directory.is_dir(): + raise SystemExit( + f"INFINITO_EXTRA_MOUNTS: {item} is not an existing absolute directory" + ) + directories.append(directory) + return directories + + +def write_mounts_override(repo_root: Path, directories: list[Path]) -> str: + """Write the compose override that mounts ``directories`` into the container. + + Args: + repo_root: repository root the compose files are relative to. + directories: host directories, each mounted read-only at its own path. + + Returns: + The override path relative to ``repo_root``. + """ + volumes = [ + {"type": "bind", "source": str(path), "target": str(path), "read_only": True} + for path in directories + ] + dump_yaml(repo_root / OVERRIDE, {"services": {"infinito": {"volumes": volumes}}}) + return OVERRIDE diff --git a/cli/administration/deploy/swarm/command.py b/cli/administration/deploy/swarm/command.py index 9fb76ebe47..c152223003 100644 --- a/cli/administration/deploy/swarm/command.py +++ b/cli/administration/deploy/swarm/command.py @@ -1,17 +1,23 @@ """Swarm-aware deploy CLI command. -Re-uses ``dedicated.command.parse_args`` verbatim, then rewrites -``args.id`` via the swarm closure between parse and dispatch. +Re-uses ``dedicated.command.parse_args`` verbatim, pulls the custom roles +first because the swarm closure reads roles/, then rewrites ``args.id`` +via that closure between parse and dispatch. """ from __future__ import annotations -from cli.administration.deploy.dedicated.command import parse_args, run_from_args +from cli.administration.deploy.dedicated.command import ( + parse_args, + pull_custom_roles, + run_from_args, +) from .closure import swarm_deploy_targets def main(argv: list[str] | None = None) -> int: args, passthrough, modes_spec = parse_args(argv) + pull_custom_roles(args) args.id = swarm_deploy_targets(args.id, args.inventory) return run_from_args(args, passthrough, modes_spec) diff --git a/cli/administration/inventory/validate/applications.py b/cli/administration/inventory/validate/applications.py index 99498dcd96..8d3a3d1ed6 100644 --- a/cli/administration/inventory/validate/applications.py +++ b/cli/administration/inventory/validate/applications.py @@ -3,17 +3,26 @@ from __future__ import annotations from utils.manager.credential_key import OVERRIDE_SECTION, SECRETS_KEY +from utils.roles.custom import CUSTOM_KEY, SOURCE_KEY, is_custom from .keys import recursive_keys +def _declares_custom_role(app_id: str, key: str) -> bool: + return is_custom(app_id) and key.split(".", 1)[0] in (CUSTOM_KEY, SOURCE_KEY) + + def compare_application_keys(applications, application_defaults, source, variants=None): """Variants.yml is part of the role contract: a variant overlay may legitimately introduce service-keys absent from meta/services.yml (e.g. roles/web-opt-rdr-www declares services.dashboard only in meta/variants.yml). Widen the allow-set to defaults ∪ union(variant overlays) so host_vars baked by matrix-deploy from variants.yml - aren't flagged as "Missing default".""" + aren't flagged as "Missing default". + + ``custom`` and ``source`` of a custom role are the inventory's + declaration of where the role comes from, not role settings, so no + role default carries them.""" variants = variants or {} errs: list[str] = [] for app_id, conf in applications.items(): @@ -26,6 +35,8 @@ def compare_application_keys(applications, application_defaults, source, variant for key in recursive_keys(conf): if key == SECRETS_KEY or key.startswith(OVERRIDE_SECTION): continue + if _declares_custom_role(app_id, key): + continue if key not in legal_keys: errs.append(f"{source}: Missing default for {app_id}: {key}") return errs diff --git a/cli/administration/inventory/validate/loaders.py b/cli/administration/inventory/validate/loaders.py index 91726607fd..6f695a8f21 100644 --- a/cli/administration/inventory/validate/loaders.py +++ b/cli/administration/inventory/validate/loaders.py @@ -26,23 +26,29 @@ def load_yaml_file(path): return None +def inventory_files(inv_dir) -> list[Path]: + """Return the YAML files of an inventory directory that may set ``applications``. + + Args: + inv_dir: the inventory directory; its ``*.yml`` and every ``*.yml`` + under its ``*_vars`` directories are listed. + """ + p = Path(inv_dir) + files = list(p.glob("*.yml")) + for d in p.glob("*_vars"): + if d.is_dir(): + files.extend(d.rglob("*.yml")) + return files + + def load_inventory_files(inv_dir) -> dict[str, dict]: """Walk the inventory directory and collect every ``applications:`` block keyed by its source file path.""" all_data: dict[str, dict] = {} - p = Path(inv_dir) - for f in p.glob("*.yml"): + for f in inventory_files(inv_dir): data = load_yaml_file(f) if isinstance(data, dict): apps = data.get("applications") if apps: all_data[str(f)] = apps - for d in p.glob("*_vars"): - if d.is_dir(): - for f in d.rglob("*.yml"): - data = load_yaml_file(f) - if isinstance(data, dict): - apps = data.get("applications") - if apps: - all_data[str(f)] = apps return all_data diff --git a/cli/administration/roles/README.md b/cli/administration/roles/README.md new file mode 100644 index 0000000000..4e1ad83c7b --- /dev/null +++ b/cli/administration/roles/README.md @@ -0,0 +1,3 @@ +# Roles 🧩 + +Custom roles (`web-app-cst-*`) an inventory declares, pulled from their own repositories into the role search path of this checkout, listed and removed again. diff --git a/cli/administration/roles/__init__.py b/cli/administration/roles/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cli/administration/roles/clean/__init__.py b/cli/administration/roles/clean/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cli/administration/roles/clean/__main__.py b/cli/administration/roles/clean/__main__.py new file mode 100644 index 0000000000..2e95391b10 --- /dev/null +++ b/cli/administration/roles/clean/__main__.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import argparse + +from cli.administration.roles.paths import WORK_DIR +from cli.administration.roles.placement import clean +from utils.cache import ROLES_DIR + + +def main(argv: list[str] | None = None) -> int: + argparse.ArgumentParser( + description=( + "Remove every custom role (roles/web-app-cst-*) a pull placed in this " + "checkout and nothing else; report every other roles/web-app-cst-* entry." + ) + ).parse_args(argv) + try: + lines = clean(ROLES_DIR, WORK_DIR) + except OSError as exc: + raise SystemExit(f"[ERROR] {exc}") from exc + for line in lines: + print(f"custom role {line}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/cli/administration/roles/declaration.py b/cli/administration/roles/declaration.py new file mode 100644 index 0000000000..fe81a16ce9 --- /dev/null +++ b/cli/administration/roles/declaration.py @@ -0,0 +1,248 @@ +"""Custom-role declarations of an inventory, validated before anything is pulled. + +``custom`` and ``source`` are read from the raw inventory files and are +never rendered: a templated or tagged value could otherwise make the +controller fetch code chosen at render time. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from urllib.parse import urlsplit + +from utils.cache.yaml import TaggedValue +from utils.roles.custom import CUSTOM_KEY, PREFIX, SOURCE_KEY, is_custom +from utils.roles.entity.name import get_entity_name + +from .inventory import read_inventory + +SOURCE_FIELDS: tuple[str, ...] = ("url", "version", "path") + +_JINJA = re.compile(r"{{|{%|{#") +_FULL_SHA = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") +_GIT_URL = re.compile(r"^(?:(?:https|ssh)://\S+|(?:[\w.-]+@)?[\w.-]+:(?!//)\S+)$") +_CONFLICT = object() + + +@dataclass(frozen=True) +class Source: + """Where a custom role comes from. + + Args: + url: Git URL over HTTPS or SSH, or an absolute local directory. + version: tag or full commit SHA; ``None`` copies the working tree + of a local directory. + path: the role directory inside the repository. + """ + + url: str + version: str | None + path: str + + @property + def local(self) -> bool: + return Path(self.url).is_absolute() + + @property + def commit(self) -> bool: + return self.version is not None and bool(_FULL_SHA.match(self.version)) + + def describe(self) -> str: + """Return the source as the pull reports it, without the resolved commit.""" + if self.version is None: + return f"{self.url} {self.path} (working tree)" + return f"{self.url} {self.path} @ {self.version}" + + +def read_declarations( + inventory: Path, roles_dir: Path +) -> tuple[dict[str, Source], list[str]]: + """Return the declared custom roles and every declaration error. + + Args: + inventory: the inventory file, whose directory holds the + ``group_vars``/``host_vars`` that set application values, or + an inventory directory. + roles_dir: the core ``roles/`` the custom roles are placed in. + + Returns: + ``({group: Source}, errors)``; each error names its group, or the + inventory file that cannot be read, which declares nothing. + """ + documents, groups, errors = read_inventory(inventory) + if errors: + return {}, errors + marked: set[str] = set() + declared: dict[str, Source] = {} + for group, by_key in sorted(_collect(documents).items()): + values = { + key: _agreed(group, key, by_file, errors) for key, by_file in by_key.items() + } + if any(value is _CONFLICT for value in values.values()): + continue + custom = values.get(CUSTOM_KEY) + if reason := _unrendered(custom): + errors.append( + f"{group}: applications.{group}.{CUSTOM_KEY} {reason}; the pull never renders it" + ) + continue + if custom is not True: + continue + marked.add(group) + if not is_custom(group): + errors.append( + f"{group}: {CUSTOM_KEY}: true marks a group outside the {PREFIX} prefix" + ) + continue + source = _source(group, values.get(SOURCE_KEY), errors) + if source is not None: + declared[group] = source + errors.extend( + f"{group}: inventory group without applications.{group}.{CUSTOM_KEY}: true" + for group in sorted(groups) + if is_custom(group) and group not in marked + ) + errors.extend(_entity_collisions(declared, roles_dir)) + return declared, errors + + +def _collect(documents: dict[str, object]) -> dict[str, dict[str, dict[str, object]]]: + found: dict[str, dict[str, dict[str, object]]] = {} + for src, data in documents.items(): + apps = data.get("applications") if isinstance(data, dict) else None + if not isinstance(apps, dict): + continue + for group, conf in apps.items(): + if not isinstance(conf, dict): + continue + for key in (CUSTOM_KEY, SOURCE_KEY): + if key in conf: + found.setdefault(str(group), {}).setdefault(key, {})[src] = conf[ + key + ] + return found + + +def _agreed( + group: str, key: str, by_file: dict[str, object], errors: list[str] +) -> object: + values = list(by_file.values()) + if all(value == values[0] for value in values[1:]): + return values[0] + errors.append( + f"{group}: applications.{group}.{key} differs between {', '.join(sorted(by_file))}" + ) + return _CONFLICT + + +def _unrendered(value: object) -> str | None: + if isinstance(value, TaggedValue): + return f"holds a {value.tag} value" + if isinstance(value, str): + return "holds a Jinja expression" if _JINJA.search(value) else None + if isinstance(value, dict): + value = list(value.values()) + if isinstance(value, list): + return next(filter(None, map(_unrendered, value)), None) + return None + + +def _source(group: str, raw: object, errors: list[str]) -> Source | None: + prefix = f"{group}: applications.{group}.{SOURCE_KEY}" + if raw is None: + errors.append(f"{prefix} is missing") + return None + if isinstance(raw, TaggedValue): + errors.append(f"{prefix} holds a {raw.tag} value; the pull never renders it") + return None + if not isinstance(raw, dict): + errors.append(f"{prefix} must be a mapping of {', '.join(SOURCE_FIELDS)}") + return None + before = len(errors) + errors.extend( + f"{prefix}.{name} is not a source field" + for name in sorted(set(raw) - set(SOURCE_FIELDS)) + ) + fields: dict[str, str] = {} + for name in SOURCE_FIELDS: + value = raw.get(name) + if value is None: + continue + if reason := _unrendered(value): + errors.append(f"{prefix}.{name} {reason}; the pull never renders it") + elif not isinstance(value, str) or not value.strip(): + errors.append(f"{prefix}.{name} must be a non-empty string") + else: + fields[name] = value + url, version, path = (fields.get(name) for name in SOURCE_FIELDS) + if url is None and "url" not in raw: + errors.append(f"{prefix}.url is missing") + if url is not None: + errors.extend(_url_errors(prefix, url)) + if ( + version is None + and "version" not in raw + and not (url is not None and Path(url).is_absolute()) + ): + errors.append(f"{prefix}.version is missing") + if path is None and "path" not in raw: + errors.append(f"{prefix}.path is missing") + if path is not None: + errors.extend(_path_errors(group, prefix, PurePosixPath(path))) + if len(errors) > before or url is None or path is None: + return None + return Source(url=url, version=version, path=path) + + +def _url_errors(prefix: str, url: str) -> list[str]: + if _embeds_credentials(url): + return [ + ( + f"{prefix}.url embeds credentials; authenticate through the SSH " + "agent or SSH configuration of the controller instead" + ) + ] + if not (Path(url).is_absolute() or _GIT_URL.match(url)): + return [ + f"{prefix}.url is neither a Git URL over HTTPS or SSH nor an absolute local directory" + ] + return [] + + +def _embeds_credentials(url: str) -> bool: + if "://" not in url: + return False + try: + parts = urlsplit(url) + password = parts.password + except ValueError: + return True + return password is not None or ( + parts.scheme != "ssh" and parts.username is not None + ) + + +def _path_errors(group: str, prefix: str, path: PurePosixPath) -> list[str]: + # nocheck: project-root-import -- refuses a source.path that climbs out of its repository + if path.is_absolute() or ".." in path.parts: + return [f"{prefix}.path must be a relative path inside the repository"] + if path.name != group: + return [ + f"{prefix}.path names the role directory '{path.name}', but the group is '{group}'; core never renames a role" + ] + return [] + + +def _entity_collisions(declared: dict[str, Source], roles_dir: Path) -> list[str]: + in_tree = { + get_entity_name(role.name): role.name + for role in sorted(roles_dir.iterdir()) + if role.is_dir() and not is_custom(role.name) + } + return [ + f"{group}: entity name '{entity}' collides with the in-tree role {in_tree[entity]}" + for group in sorted(declared) + if (entity := get_entity_name(group)) in in_tree + ] diff --git a/cli/administration/roles/inventory.py b/cli/administration/roles/inventory.py new file mode 100644 index 0000000000..e6f56991e0 --- /dev/null +++ b/cli/administration/roles/inventory.py @@ -0,0 +1,105 @@ +"""The inventory files a custom-role pull reads, picked the way Ansible picks them. + +A pull stops while any of these files cannot be read: a declaration it +cannot see would make it remove the role as no longer declared. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +from yaml import MarkedYAMLError, YAMLError + +from cli.administration.inventory.validate.loaders import inventory_files +from utils.cache.yaml import load_yaml_tagged +from utils.inventory.groups import inventory_groups + +if TYPE_CHECKING: + from pathlib import Path + +_SKIPPED_NAMES = re.compile(r"^\.|^(?:host_vars|group_vars|vars_plugins)$") +_SKIPPED_SUFFIXES = ( + ".pyc", + ".pyo", + ".swp", + ".bak", + "~", + ".rpm", + ".md", + ".txt", + ".rst", + ".orig", + ".cfg", + ".retry", +) +_UNREADABLE = (OSError, ValueError, YAMLError) + + +def is_host_list(inventory: Path) -> bool: + """Return whether *inventory* is an inline host list such as ``localhost,``. + + Args: + inventory: the ``-i`` value; Ansible reads it as a host list when + no such path exists and it holds a comma. + """ + return "," in str(inventory) and not inventory.exists() + + +def read_inventory(inventory: Path) -> tuple[dict[str, object], set[str], list[str]]: + """Read every file of *inventory* the pull takes declarations and groups from. + + Args: + inventory: an inventory file, or a directory Ansible reads as one: + its host files, skipped the way Ansible skips them, and the + ``group_vars``/``host_vars`` beside them. + + Returns: + ``({file: document}, groups, errors)``; each error names a file + that cannot be read. Tagged values load as opaque + :class:`~utils.cache.yaml.TaggedValue` objects. + """ + if inventory.is_dir(): + base, hosts = inventory, _host_files(inventory) + else: + base, hosts = inventory.parent, [inventory] + documents: dict[str, object] = {} + errors: list[str] = [] + for path in inventory_files(base): + try: + documents[str(path)] = load_yaml_tagged(path) + except _UNREADABLE as exc: + errors.append(_unreadable(path, exc)) + groups: set[str] = set() + for path in hosts: + try: + groups |= inventory_groups(str(path)) + except _UNREADABLE as exc: + if (message := _unreadable(path, exc)) not in errors: + errors.append(message) + return documents, groups, errors + + +def _host_files(directory: Path) -> list[Path]: + files: list[Path] = [] + for entry in sorted(directory.iterdir()): + if _SKIPPED_NAMES.search(entry.name) or entry.name.endswith(_SKIPPED_SUFFIXES): + continue + if entry.is_dir(): + files.extend(_host_files(entry)) + elif entry.is_file(): + files.append(entry) + return files + + +def _unreadable(path: Path, exc: Exception) -> str: + if isinstance(exc, UnicodeDecodeError): + reason = "it is not UTF-8" + elif isinstance(exc, MarkedYAMLError) and exc.problem_mark is not None: + reason = f"{exc.problem} at line {exc.problem_mark.line + 1}" + else: + reason = str(exc) + return ( + f"{path}: cannot read the inventory file ({reason}); the pull changes " + "nothing while an inventory file is unreadable" + ) diff --git a/cli/administration/roles/list/__init__.py b/cli/administration/roles/list/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cli/administration/roles/list/__main__.py b/cli/administration/roles/list/__main__.py new file mode 100644 index 0000000000..1f0200c5e6 --- /dev/null +++ b/cli/administration/roles/list/__main__.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import argparse + +from cli.administration.roles.marker import placed_roles, provenance +from utils.cache import ROLES_DIR + + +def main(argv: list[str] | None = None) -> int: + argparse.ArgumentParser( + description=( + "Print the custom roles a pull placed under roles/ of this checkout, " + "one per line with its source and resolved commit." + ) + ).parse_args(argv) + for role in placed_roles(ROLES_DIR): + print(f"{role.name}: {provenance(role)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/cli/administration/roles/marker.py b/cli/administration/roles/marker.py new file mode 100644 index 0000000000..3dbf05163c --- /dev/null +++ b/cli/administration/roles/marker.py @@ -0,0 +1,118 @@ +"""The marker that identifies a custom role the pull placed under ``roles/``. + +The marker names the directory it was placed as, so a copy of a placed +role under another name, which carries the same file, never counts as +placed. It is written and read without following a symbolic link. +""" + +from __future__ import annotations + +import os +import shutil +import stat +from typing import TYPE_CHECKING + +from utils.roles.custom import PREFIX + +if TYPE_CHECKING: + from pathlib import Path + +MARKER_FILE = ".infinito-custom-role" +MARKER_HEADER = "# infinito-nexus custom role" +_MARKER_LIMIT = 4096 + + +def marker_text(directory: str, group: str, provenance: str) -> str: + """Return the content of the marker file. + + Args: + directory: the name the role directory is placed as under ``roles/``. + group: the inventory group the role was pulled for. + provenance: the source and resolved commit the role was placed from. + """ + return ( + f"{MARKER_HEADER}\ndirectory: {directory}\ngroup: {group}\n" + f"source: {provenance}\n" + ) + + +def write_marker(role: Path, group: str, provenance: str) -> None: + """Write the marker into a staged role, replacing whatever entry holds its name. + + The entry is removed without following it and the marker is created + exclusively, so a symbolic link shipped under that name never + redirects the write. + + Args: + role: the staged role directory, named as it is placed. + group: the inventory group the role was pulled for. + provenance: the source and resolved commit the role was placed from. + """ + marker = role / MARKER_FILE + try: + mode = marker.lstat().st_mode + except FileNotFoundError: + mode = None + if mode is not None and stat.S_ISDIR(mode): + shutil.rmtree(marker) + elif mode is not None: + marker.unlink() + descriptor = os.open( + marker, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644 + ) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(marker_text(role.name, group, provenance)) + + +def is_placed(path: Path) -> bool: + """Return whether the pull placed ``path``: its marker names ``path`` itself. + + Args: + path: an entry of the role search directory. + """ + return not path.is_symlink() and _fields(path).get("directory") == path.name + + +def provenance(path: Path) -> str: + """Return the source and resolved commit recorded for a placed role. + + Args: + path: a role directory :func:`is_placed` accepts. + """ + return _fields(path).get("source", "") + + +def custom_entries(roles_dir: Path) -> list[Path]: + """Return every ``roles/web-app-cst-*`` entry, placed or not, sorted. + + Args: + roles_dir: the role search directory. + """ + return sorted(roles_dir.glob(f"{PREFIX}*")) + + +def placed_roles(roles_dir: Path) -> list[Path]: + """Return the custom roles the pull placed under ``roles_dir``, sorted. + + Args: + roles_dir: the role search directory. + """ + return [entry for entry in custom_entries(roles_dir) if is_placed(entry)] + + +def _fields(role: Path) -> dict[str, str]: + # Exception: O_NONBLOCK keeps a FIFO planted under the marker name from + # blocking the open; the fstat below refuses it. + try: + descriptor = os.open( + role / MARKER_FILE, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + ) + except OSError: + return {} + with os.fdopen(descriptor, encoding="utf-8", errors="replace") as handle: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + return {} + lines = handle.read(_MARKER_LIMIT).splitlines() + if not lines or lines[0] != MARKER_HEADER: + return {} + return dict(line.split(": ", 1) for line in lines[1:] if ": " in line) diff --git a/cli/administration/roles/paths.py b/cli/administration/roles/paths.py new file mode 100644 index 0000000000..17d4362eec --- /dev/null +++ b/cli/administration/roles/paths.py @@ -0,0 +1,7 @@ +"""Where custom roles are staged before the pull moves them under ``roles/``.""" + +from __future__ import annotations + +from utils.cache import PROJECT_ROOT + +WORK_DIR = PROJECT_ROOT / "build" / "custom-roles" diff --git a/cli/administration/roles/placement.py b/cli/administration/roles/placement.py new file mode 100644 index 0000000000..631cfc6064 --- /dev/null +++ b/cli/administration/roles/placement.py @@ -0,0 +1,233 @@ +"""Place the custom roles an inventory declares under ``roles/`` and remove them again. + +A pull stages every declared role in a work directory outside ``roles/`` +and checks it before the first entry of ``roles/`` changes, so a refused +pull leaves ``roles/`` as it was. A pull or clean only ever replaces or +removes the directories the pull placed itself, whose marker names them. + +Both hold an exclusive lock under the work directory, so two of them on +one checkout never interleave, and clear what an interrupted run left +there. They swap entries by rename and delete the old copies only after +the swap, so an interrupt never leaves a declared role missing. +""" + +from __future__ import annotations + +import contextlib +import fcntl +import os +import shutil +import sys +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING + +from yaml import YAMLError + +from utils.cache.yaml import load_yaml_tagged +from utils.roles.custom import CustomRoleError +from utils.roles.mapping import ROLE_FILE_META_MAIN, ROLE_FILE_VARS_MAIN + +from .declaration import read_declarations +from .inventory import is_host_list +from .marker import MARKER_FILE, custom_entries, is_placed, write_marker +from .source import check, stage + +if TYPE_CHECKING: + from collections.abc import Iterator + + from .declaration import Source + +LOCK_FILE = ".lock" +_LEFT = "left in place, the pull did not place it" + + +def pull(inventory: Path, roles_dir: Path, work_dir: Path) -> list[str]: + """Place the custom roles *inventory* declares and remove the ones it dropped. + + Args: + inventory: the inventory file or directory; an inline host list + such as ``localhost,`` declares nothing and changes nothing. + roles_dir: the ``roles/`` of the core checkout that runs the deploy. + work_dir: staging directory outside ``roles_dir`` on its filesystem. + + Returns: + One line per placed or removed role, naming its source and + resolved commit, and one per ``roles/web-app-cst-*`` entry the + pull left alone. + + Raises: + CustomRoleError: naming the group, or the file, that stops the run. + """ + if is_host_list(inventory): + return [] + if not inventory.exists(): + raise CustomRoleError( + f"inventory {inventory} is neither a file nor a directory" + ) + declared, errors = read_declarations(inventory, roles_dir) + if errors: + raise CustomRoleError("\n".join(errors)) + if not declared and not custom_entries(roles_dir): + return [] + with _locked(work_dir): + return _place(declared, roles_dir, work_dir) + + +def clean(roles_dir: Path, work_dir: Path) -> list[str]: + """Remove every ``roles/web-app-cst-*`` directory the pull placed and nothing else. + + Args: + roles_dir: the role search directory. + work_dir: staging directory outside ``roles_dir`` on its filesystem. + + Returns: + One line per ``roles/web-app-cst-*`` entry: removed, or left in + place because the pull did not place it. + """ + with _locked(work_dir): + entries = custom_entries(roles_dir) + placed = [entry for entry in entries if is_placed(entry)] + holders: list[Path] = [] + try: + _rename_all( + [(entry, _retired(entry, work_dir, holders)) for entry in placed] + ) + finally: + _discard(holders) + return [ + f"{entry.name}: removed" if entry in placed else f"{entry.name}: {_LEFT}" + for entry in entries + ] + + +def _place(declared: dict[str, Source], roles_dir: Path, work_dir: Path) -> list[str]: + entries = custom_entries(roles_dir) + for group, source in sorted(declared.items()): + check(group, source, entries, roles_dir) + _check_target(roles_dir / group) + undeclared = [entry for entry in entries if entry.name not in declared] + lines: list[str] = [] + staged: list[tuple[Path, Path]] = [] + holders: list[Path] = [] + try: + for group, source in sorted(declared.items()): + staging = _holder(work_dir, group, holders) / group + origin = stage(group, source, staging) + _check_role(group, staging) + write_marker(staging, group, origin) + staged.append((staging, roles_dir / group)) + lines.append(f"{group}: {origin}") + for _staging, target in staged: + _check_target(target) + dropped = [entry for entry in undeclared if is_placed(entry)] + moves = [(entry, _retired(entry, work_dir, holders)) for entry in dropped] + for staging, target in staged: + if os.path.lexists(target): + moves.append((target, _retired(target, work_dir, holders))) + moves.append((staging, target)) + _rename_all(moves) + finally: + _discard(holders) + lines.extend( + f"{entry.name}: removed, the inventory no longer declares it" + if entry in dropped + else f"{entry.name}: {_LEFT}" + for entry in undeclared + ) + return sorted(lines) + + +def _check_target(target: Path) -> None: + if target.is_symlink(): + raise CustomRoleError( + f"{target.name}: {target} is a symbolic link to {target.readlink()}, " + f"not a role the pull placed; remove the link (rm {target}) and pull again" + ) + if target.exists() and not is_placed(target): + raise CustomRoleError( + f"{target.name}: {target} was not placed by a pull (its {MARKER_FILE} " + "does not name it); the pull never replaces it, remove it by hand" + ) + + +def _check_role(group: str, staged: Path) -> None: + if (staged / MARKER_FILE).is_symlink(): + raise CustomRoleError( + f"{group}: source.path {MARKER_FILE} is a symbolic link; the pull " + "refuses a source that could redirect the marker it writes" + ) + if not (staged / ROLE_FILE_META_MAIN).is_file(): + raise CustomRoleError( + f"{group}: source.path holds no {ROLE_FILE_META_MAIN}, so it is no role directory" + ) + variables = staged / ROLE_FILE_VARS_MAIN + try: + data = load_yaml_tagged(variables) if variables.is_file() else None + except (ValueError, YAMLError) as exc: + raise CustomRoleError( + f"{group}: {ROLE_FILE_VARS_MAIN} cannot be read: {exc}" + ) from exc + application_id = data.get("application_id") if isinstance(data, dict) else None + if application_id != group: + raise CustomRoleError( + f"{group}: {ROLE_FILE_VARS_MAIN} application_id is {application_id!r}, but the group is '{group}'; core never renames a role" + ) + + +@contextlib.contextmanager +def _locked(work_dir: Path) -> Iterator[None]: + work_dir.mkdir(parents=True, exist_ok=True) + lock = work_dir / LOCK_FILE + descriptor = os.open(lock, os.O_RDONLY | os.O_CREAT | os.O_NOFOLLOW, 0o644) + try: + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print( + f"[INFO] waiting for another custom-role pull or clean of this checkout ({lock})", + file=sys.stderr, + flush=True, + ) + fcntl.flock(descriptor, fcntl.LOCK_EX) + _discard([entry for entry in work_dir.iterdir() if entry.name != LOCK_FILE]) + yield + finally: + os.close(descriptor) + + +def _holder(work_dir: Path, prefix: str, holders: list[Path]) -> Path: + holder = Path(tempfile.mkdtemp(prefix=f"{prefix}.", dir=work_dir)) + holders.append(holder) + return holder + + +def _retired(entry: Path, work_dir: Path, holders: list[Path]) -> Path: + return _holder(work_dir, f"{entry.name}.retired", holders) / entry.name + + +def _rename_all(moves: list[tuple[Path, Path]]) -> None: + done: list[tuple[Path, Path]] = [] + try: + for old, new in moves: + old.rename(new) + done.append((old, new)) + except BaseException: + for old, new in reversed(done): + new.rename(old) + raise + + +def _discard(entries: list[Path]) -> None: + for entry in entries: + try: + if entry.is_dir() and not entry.is_symlink(): + shutil.rmtree(entry) + else: + entry.unlink() + except OSError as exc: + print( + f"[WARN] cannot remove {entry} ({exc}); container-owned leftovers " + "go with make clean-container-owned", + file=sys.stderr, + ) diff --git a/cli/administration/roles/pull/__init__.py b/cli/administration/roles/pull/__init__.py new file mode 100644 index 0000000000..a50a1bac8c --- /dev/null +++ b/cli/administration/roles/pull/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from .command import main, pull_or_exit + +__all__ = ["main", "pull_or_exit"] diff --git a/cli/administration/roles/pull/__main__.py b/cli/administration/roles/pull/__main__.py new file mode 100644 index 0000000000..32f6bc12c4 --- /dev/null +++ b/cli/administration/roles/pull/__main__.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +from .command import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/cli/administration/roles/pull/command.py b/cli/administration/roles/pull/command.py new file mode 100644 index 0000000000..b04cb5386e --- /dev/null +++ b/cli/administration/roles/pull/command.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import argparse +import re +import sys +import tarfile +from pathlib import Path + +from yaml import YAMLError + +from cli.administration.roles.paths import WORK_DIR +from cli.administration.roles.placement import pull +from utils.cache import ROLES_DIR +from utils.roles.custom import CustomRoleError + +_URL_USERINFO = re.compile(r"(?<=://)[^/\s@'\"]+@") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Pull the custom roles (web-app-cst-*) an inventory declares into " + "roles/, replace the ones an earlier pull placed and remove the ones " + "it no longer declares, without deploying." + ), + ) + parser.add_argument( + "inventory", + help="Inventory file or directory, or an inline host list such as 'localhost,'.", + ) + return parser + + +def pull_or_exit(inventory: str) -> list[str]: + """Pull the custom roles of *inventory*; exit 1 when the pull is refused. + + The reason never carries the credentials of a URL. + + Args: + inventory: the inventory file or directory, or an inline host list. + + Returns: + One line per placed, removed or untouched custom role; none and + no output when nothing is declared or placed. + """ + try: + lines = pull(Path(inventory), ROLES_DIR, WORK_DIR) + except (CustomRoleError, OSError, tarfile.TarError, YAMLError) as exc: + reason = _URL_USERINFO.sub("***@", str(exc)) + print(f"\n[ERROR] Custom roles refused:\n{reason}\n", file=sys.stderr) + sys.exit(1) + for line in lines: + print(f"custom role {line}", flush=True) + return lines + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + pull_or_exit(args.inventory) + return 0 diff --git a/cli/administration/roles/source.py b/cli/administration/roles/source.py new file mode 100644 index 0000000000..cdeba91acb --- /dev/null +++ b/cli/administration/roles/source.py @@ -0,0 +1,177 @@ +"""Check the source of a custom role and stage its content.""" + +from __future__ import annotations + +import io +import os +import shutil +import subprocess +import tarfile +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING + +from utils.roles.custom import CustomRoleError + +if TYPE_CHECKING: + from .declaration import Source + +_COPY_IGNORE = shutil.ignore_patterns(".git", "__pycache__") +_DROPPED_ENV = frozenset( + { + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_ASKPASS", + "GIT_COMMON_DIR", + "GIT_DIR", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_WORK_TREE", + "SSH_ASKPASS", + } +) + + +def check(group: str, source: Source, placements: list[Path], roles_dir: Path) -> None: + """Refuse a source before anything is fetched. + + Args: + group: the custom role's inventory group. + source: its validated declaration. + placements: every ``roles/web-app-cst-*`` entry that exists now. + roles_dir: the core ``roles/`` the role is placed in. + + Raises: + CustomRoleError: for a missing local directory, a local source the + pull would delete, or a version that is no tag or full commit SHA. + """ + if source.local: + _check_local(group, source, [roles_dir / group, *placements], roles_dir) + if source.version is None or source.commit: + return + refs = { + line.split("\t", 1)[1] + for line in _git( + None, ["ls-remote", "--", source.url], f"{group}: git ls-remote failed" + ) + .decode(errors="replace") + .splitlines() + if "\t" in line + } + if f"refs/heads/{source.version}" in refs: + raise CustomRoleError( + f"{group}: source.version '{source.version}' names a branch; pin a tag or a full commit SHA" + ) + if f"refs/tags/{source.version}" not in refs: + raise CustomRoleError( + f"{group}: source.version '{source.version}' is neither a tag nor a full commit SHA" + ) + + +def stage(group: str, source: Source, target: Path) -> str: + """Copy the role of ``source`` into ``target`` and describe what was copied. + + Args: + group: the custom role's inventory group. + source: a checked declaration. + target: a path that does not exist yet; it becomes the role directory. + + Returns: + The source and, for a pinned version, the resolved commit. + + Raises: + CustomRoleError: the source cannot be read, or it holds an entry, + such as a link to an absolute path, the pull refuses to unpack. + """ + if source.version is None: + content = Path(source.url) / source.path + if not content.is_dir(): + raise CustomRoleError( + f"{group}: source.path is not a directory in the source" + ) + shutil.copytree(content, target, symlinks=True, ignore=_COPY_IGNORE) + return source.describe() + with tempfile.TemporaryDirectory(prefix="infinito-custom-role-") as scratch: + _git(scratch, ["init", "--quiet", "--bare"], f"{group}: git init failed") + _git( + scratch, + [ + "fetch", + "--quiet", + "--depth", + "1", + "--no-tags", + "--", + source.url, + source.version, + ], + f"{group}: git fetch of {source.version} failed", + ) + commit = ( + _git( + scratch, + ["rev-parse", "--verify", "FETCH_HEAD^{commit}"], + f"{group}: {source.version} names no commit", + ) + .decode() + .strip() + ) + tree = f"{commit}:{source.path}" + kind = _git( + scratch, + ["cat-file", "-t", tree], + f"{group}: source.path is not a directory in the source", + ) + if kind.decode().strip() != "tree": + raise CustomRoleError( + f"{group}: source.path is not a directory in the source" + ) + archive = _git( + scratch, + ["archive", "--format=tar", tree], + f"{group}: git archive of source.path failed", + ) + target.mkdir() + try: + with tarfile.open(fileobj=io.BytesIO(archive)) as tar: + tar.extractall(target, filter="data") + except tarfile.TarError as exc: + raise CustomRoleError( + f"{group}: source.path holds an entry the pull refuses to unpack: {exc}" + ) from exc + return f"{source.describe()} (commit {commit})" + + +def _check_local( + group: str, source: Source, placements: list[Path], roles_dir: Path +) -> None: + if not Path(source.url).is_dir(): + raise CustomRoleError(f"{group}: source.url {source.url} is not a directory") + role = (Path(source.url) / source.path).resolve() + for placement in placements: + resolved = placement.resolve() + if role == resolved or role.is_relative_to(resolved): + raise CustomRoleError( + f"{group}: local source {role} lies in the placement directory {placement} under {roles_dir}; a pull never deletes its own source" + ) + + +def _git(cwd: str | None, arguments: list[str], failure: str) -> bytes: + # Exception: a git hook exports GIT_DIR, GIT_INDEX_FILE and friends, which + # would point the scratch repository at the core checkout; the askpass + # helpers and the detached session keep a fetch that needs credentials + # from prompting. + env = {key: value for key, value in os.environ.items() if key not in _DROPPED_ENV} + env.update(GIT_TERMINAL_PROMPT="0", SSH_ASKPASS_REQUIRE="never") + result = subprocess.run( + ["git", *arguments], + cwd=cwd, + env=env, + stdin=subprocess.DEVNULL, + capture_output=True, + start_new_session=True, + check=False, + ) + if result.returncode != 0: + detail = result.stderr.decode(errors="replace").strip() + raise CustomRoleError(f"{failure}: {detail}") + return result.stdout diff --git a/cli/build/docs/readme/cosmos.py b/cli/build/docs/readme/cosmos.py index 3f2abcc2d0..51639377b3 100644 --- a/cli/build/docs/readme/cosmos.py +++ b/cli/build/docs/readme/cosmos.py @@ -11,6 +11,9 @@ The role box lists EVERY service the role declares. Around it: above: another role names this one, or declares a service keyed by this role's ``provides`` name (e.g. web-app-keycloak ``provides: sso`` and every role with an ``sso`` service). + An in-tree role never lists a custom role (``web-app-cst-*``) + here: Git ignores custom roles, so a tracked README never + names one. ``provides`` matching runs in both directions, so a provider (e.g. openldap ``provides: ldap``) is picked up even when the consuming flag uses a @@ -44,6 +47,7 @@ from utils.roles.applications.services.registry import ( build_service_registry_from_roles_dir, is_explicit_truth, ) +from utils.roles.custom import is_custom from utils.roles.deploy import role_deploy_modes from utils.roles.mapping import ROLE_FILE_META_MAIN, ROLE_FILE_META_SERVICES from utils.symbol_glossary import to_emoji @@ -299,7 +303,7 @@ def derive_cosmos_mermaid(role_dir: Path, role_name: str) -> str: dpt_kind: dict[str, str] = {} gear_dependents: set[str] = set() for other, other_services in meta.items(): - if other == role_name: + if other == role_name or (is_custom(other) and not is_custom(role_name)): continue kind = _consumption_kind(other_services, role_name, my_provides, my_primary_key) if role_name in ansible_deps.get(other, set()): diff --git a/cli/build/docs/readme/overview.py b/cli/build/docs/readme/overview.py index 1c37145e11..508f38adaa 100644 --- a/cli/build/docs/readme/overview.py +++ b/cli/build/docs/readme/overview.py @@ -9,7 +9,8 @@ run; every other section stays untouched. Rows are the invokable roles, sorted ascending by the Entity column. Rows are the invokable roles inside the tested lifecycle envelope, sorted -ascending by name. +ascending by name. Custom roles (``web-app-cst-*``) are left out: Git ignores +them, so the tracked README never names one. Columns: Name Role README H1 title, linked to the role directory. @@ -32,6 +33,7 @@ from pathlib import Path from cli.meta.roles.applications.complexity.model import compute_complexity_rows from utils.cache.files import PROJECT_ROOT, read_text from utils.cache.yaml import load_yaml_any +from utils.roles.custom import is_custom from utils.roles.entity.name import get_entity_name from utils.roles.lifecycle import tested_lifecycles from utils.roles.mapping import ( @@ -109,7 +111,9 @@ def build_table(roles_dir: Path) -> str: roles = sorted( p.name for p in roles_dir.iterdir() - if p.is_dir() and _is_role_invokable(p.name, invokable_paths) + if p.is_dir() + and not is_custom(p.name) + and _is_role_invokable(p.name, invokable_paths) ) complexity = {row.name: row for row in compute_complexity_rows(roles_dir)} roles = [ diff --git a/default.env b/default.env index f5eea27d51..668e0fb291 100644 --- a/default.env +++ b/default.env @@ -190,6 +190,9 @@ INFINITO_DOCKER_VOLUME=docker # Top-level project domain rendered into vhost/cert/test fixtures; CI/dev default, overridden in real deploys. INFINITO_DOMAIN=infinito.test +# Comma-separated absolute host directories mounted read-only at the same path into the infinito container; empty mounts none. +INFINITO_EXTRA_MOUNTS= + # Repo-relative path of the filesystem userland install script (ext4, btrfs, zfs). INFINITO_FILESYSTEM_INSTALL_SCRIPT=scripts/install/filesystem.sh diff --git a/docs/administration/deploy.md b/docs/administration/deploy.md index 94b56fea01..07a13a237f 100644 --- a/docs/administration/deploy.md +++ b/docs/administration/deploy.md @@ -40,5 +40,56 @@ When you need to redeploy after changes, repeat the same two `infinito` commands - Update `--include` whenever the target app set changes. - Use a `--vars-file` that matches the target environment. Production deploys MUST NOT point at the development sample file. +## Custom Roles 🧩 + +An inventory MAY deploy application roles that live in their own, possibly private, repositories. + +### Declare 📝 + +A custom role is an inventory group named `web-app-cst-`; a host joins it like any `web-app-*` group. Its application values mark it and name its source: + +```yaml +applications: + web-app-cst-security-check: + custom: true + source: + url: git@github.com:infinitonexus/php-landingpage.git + version: v1.4.0 + path: ansible/roles/web-app-cst-security-check +``` + +- `url` is a Git URL over HTTPS or SSH. During development an absolute local directory MAY stand in for it; without `version` its working tree is copied, uncommitted changes included. +- `version` is a tag or a full commit SHA. A branch is refused. +- `path` is the role directory inside the repository. Its name and the `application_id` in its `vars/main.yml` MUST equal the group. +- Values MUST be literal strings; Jinja is not rendered. + +### Pull 🔄 + +`infinito administration deploy dedicated` and `infinito administration deploy swarm` pull the declared roles into `roles/` of the controller's checkout before they read any role. Pass `--skip-pull` to deploy the roles already placed. The same steps run on their own: + +```bash +make roles-pull inventory=/etc/infinito.nexus/inventories/production/devices.yml +infinito administration roles list +make roles-clean +``` + +A pull stages every role outside `roles/` and changes `roles/` only when all of them passed, so a refused pull leaves it as it was. It replaces or removes only the directories it placed itself, whose marker names them, and refuses a declared name that meets a symbolic link, a hand-placed directory or a copy of a placed role. `make roles-clean` removes every `roles/web-app-cst-*` directory the pull placed and reports every other one it leaves in place. Git ignores `roles/web-app-cst-*`, so a pulled role never enters a core commit. + +For the local compose stack, `INFINITO_EXTRA_MOUNTS=[,]` mounts each directory read-only at the same path into the `infinito` container, so a local `url` resolves inside it. + +### Private Repositories 🔐 + +Git fetches with the SSH agent or the SSH configuration of the controller, so a deploy key goes into the agent or into `~/.ssh/config`: + +```text +Host github-landingpage + HostName github.com + User git + IdentityFile ~/.ssh/landingpage-deploy-key + IdentitiesOnly yes +``` + +The declaration then uses `url: github-landingpage:infinitonexus/php-landingpage.git`. A `url` that embeds a password or token is refused without being echoed, and a fetch that would ask for credentials fails instead of prompting. + For CLI installation prerequisites, see the [Installation Guide](installation.md). For the local development deploy flow (make targets, matrix variants, single-variant pinning), see [Local Deploy](../contributing/actions/deploy.md). diff --git a/docs/contributing/artefact/files/role/readme_md.md b/docs/contributing/artefact/files/role/readme_md.md index 36e5c8e417..a6fcad5eba 100644 --- a/docs/contributing/artefact/files/role/readme_md.md +++ b/docs/contributing/artefact/files/role/readme_md.md @@ -119,6 +119,8 @@ Licensed under the [Infinito.Nexus Community License (Non-Commercial)](https://s `Author name` MUST equal `galaxy_info.author` in the role's `meta/main.yml` (the author single point of truth). It MAY be wrapped in a Markdown link as shown, or left as plain bold text (`Implemented by **Author name**.`) when no author URL applies. +A custom role (`roles/web-app-cst-*`, pulled from its own repository) is exempt from the `Licensed under …` line above and states its own licence there instead; the rest of the format stays the same. + ## Formatting Rules 📏 - Role README files MUST NOT use emojis in headings. diff --git a/docs/requirements/046-custom-roles.md b/docs/requirements/046-custom-roles.md index 62a4a12c86..1506dfc159 100644 --- a/docs/requirements/046-custom-roles.md +++ b/docs/requirements/046-custom-roles.md @@ -8,36 +8,36 @@ As an operator, I want to declare application roles that live in their own repos ### Declaration -- [ ] A custom role is an ordinary inventory group named `web-app-cst-`; a host joins it the same way it joins any `web-app-*` group. -- [ ] `applications.web-app-cst-.custom: true`, set in `host_vars` or anywhere else the inventory sets application values, marks the group as a custom role. -- [ ] `applications.web-app-cst-.source` holds `url` (a Git URL over HTTPS or SSH), `version` (a tag or a full commit SHA) and `path` (the role directory inside that repository). -- [ ] A custom role whose `source.version` names a branch, or whose `source` misses `url`, `version` or `path`, stops the run before anything is pulled, and the message names the group and the missing or rejected field. -- [ ] A group named `web-app-cst-*` without `custom: true`, or `custom: true` on a group outside the `web-app-cst-` prefix, stops the run with a message naming the group. -- [ ] The pull reads `custom` and `source` from the raw inventory and never renders them as templates: a value that holds a Jinja expression stops the run with a message naming the group and the field, so no templated value can trigger a fetch. -- [ ] A `source.url` that embeds credentials stops the run without echoing the URL. +- [x] A custom role is an ordinary inventory group named `web-app-cst-`; a host joins it the same way it joins any `web-app-*` group. +- [x] `applications.web-app-cst-.custom: true`, set in `host_vars` or anywhere else the inventory sets application values, marks the group as a custom role. +- [x] `applications.web-app-cst-.source` holds `url` (a Git URL over HTTPS or SSH), `version` (a tag or a full commit SHA) and `path` (the role directory inside that repository). +- [x] A custom role whose `source.version` names a branch, or whose `source` misses `url`, `version` or `path`, stops the run before anything is pulled, and the message names the group and the missing or rejected field. +- [x] A group named `web-app-cst-*` without `custom: true`, or `custom: true` on a group outside the `web-app-cst-` prefix, stops the run with a message naming the group. +- [x] The pull reads `custom` and `source` from the raw inventory and never renders them as templates: a value that holds a Jinja expression stops the run with a message naming the group and the field, so no templated value can trigger a fetch. +- [x] A `source.url` that embeds credentials stops the run without echoing the URL. ### Authentication -- [ ] A private repository is fetched through the SSH agent or SSH configuration of the controller. -- [ ] A fetch that needs interactive credentials fails instead of prompting. -- [ ] No credential reaches the inventory, the `ansible-playbook` command line, Ansible variables or the deploy log. +- [x] A private repository is fetched through the SSH agent or SSH configuration of the controller. +- [x] A fetch that needs interactive credentials fails instead of prompting. +- [x] No credential reaches the inventory, the `ansible-playbook` command line, Ansible variables or the deploy log. ### Pull and placement -- [ ] Before the playbook starts, and so before the applications SPOT reads `roles/*/meta/`, the deploy pulls every custom role declared for the target hosts at its `source.version` and places `source.path` as `roles/web-app-cst-` in the core checkout that runs the deploy. -- [ ] A `make` target pulls the custom roles an inventory declares into the local core checkout without deploying, so every other `make` target can run against them. -- [ ] A local directory may stand in for `source.url` during development, so a role checkout is verified and deployed without a tag. -- [ ] A local source that resolves to its own placement directory `roles/web-app-cst-` stops the run with a message naming the path, so a pull never deletes its own source. -- [ ] A placed role whose directory name or `vars/main.yml` `application_id` differs from its group name stops the run with a message naming both values; core never renames a role. -- [ ] Pulling again replaces each `roles/web-app-cst-*` directory with the pinned content, and removes the directory of a custom role the inventory no longer declares. -- [ ] A pull that stops leaves `roles/` unchanged, and no staging or leftover directory of a pull appears under `roles/`. -- [ ] A declared name that meets a symbolic link under `roles/`, or a directory the pull did not place, stops the run with a message naming it, and the pull never replaces or removes such a directory. -- [ ] A command lists the placed custom roles with their source and resolved commit, and a `make` target removes every `roles/web-app-cst-*` directory and nothing else. +- [x] Before the playbook starts, and so before the applications SPOT reads `roles/*/meta/`, the deploy pulls every custom role declared for the target hosts at its `source.version` and places `source.path` as `roles/web-app-cst-` in the core checkout that runs the deploy. +- [x] A `make` target pulls the custom roles an inventory declares into the local core checkout without deploying, so every other `make` target can run against them. +- [x] A local directory may stand in for `source.url` during development, so a role checkout is verified and deployed without a tag. +- [x] A local source that resolves to its own placement directory `roles/web-app-cst-` stops the run with a message naming the path, so a pull never deletes its own source. +- [x] A placed role whose directory name or `vars/main.yml` `application_id` differs from its group name stops the run with a message naming both values; core never renames a role. +- [x] Pulling again replaces each `roles/web-app-cst-*` directory with the pinned content, and removes the directory of a custom role the inventory no longer declares. +- [x] A pull that stops leaves `roles/` unchanged, and no staging or leftover directory of a pull appears under `roles/`. +- [x] A declared name that meets a symbolic link under `roles/`, or a directory the pull did not place, stops the run with a message naming it, and the pull never replaces or removes such a directory. +- [x] A command lists the placed custom roles with their source and resolved commit, and a `make` target removes every `roles/web-app-cst-*` directory the pull placed and nothing else. - [ ] `INFINITO_EXTRA_MOUNTS=[,]` mounts each directory read-only at the same path into the `infinito` container when the stack is created, so a local `source.url` resolves inside the container; an entry that is not an existing absolute directory stops the stack start with an error naming it. -- [ ] `roles/web-app-cst-*` is ignored by Git in core, so a pulled role never enters a core commit. -- [ ] The `web-app-cst-` prefix is reserved: a lint fails on an in-tree role whose name starts with it. -- [ ] `meta/categories.yml` declares `web-app-cst` as a category, so the entity name of `web-app-cst-` is `` and the container, database and volume names a custom role derives stay what they were before its rename. -- [ ] A custom role whose entity name equals the entity name of an in-tree role stops the run with a message naming both roles. +- [x] `roles/web-app-cst-*` is ignored by Git in core, so a pulled role never enters a core commit. +- [x] The `web-app-cst-` prefix is reserved: a lint fails on an in-tree role whose name starts with it. +- [x] `meta/categories.yml` declares `web-app-cst` as a category, so the entity name of `web-app-cst-` is `` and the container, database and volume names a custom role derives stay what they were before its rename. +- [x] A custom role whose entity name equals the entity name of an in-tree role stops the run with a message naming both roles. - [ ] A swarm or Act run (`make swarm-*`, `make act-*`) carries the pulled custom roles into the copy of the core checkout its nodes use. ### Verification @@ -50,5 +50,5 @@ As an operator, I want to declare application roles that live in their own repos ### Quality -- [ ] The pull, placement, validation and cleanup logic has unit tests, including a branch name refused as `source.version`, a role whose `application_id` differs from its group, and the removal of a role the inventory no longer declares. +- [x] The pull, placement, validation and cleanup logic has unit tests, including a branch name refused as `source.version`, a role whose `application_id` differs from its group, and the removal of a role the inventory no longer declares. - [ ] `make test` passes. diff --git a/meta/categories.yml b/meta/categories.yml index a0d47de229..75e228315b 100644 --- a/meta/categories.yml +++ b/meta/categories.yml @@ -180,6 +180,13 @@ roles: run_after: [web-svc] invokable: true + cst: + title: "Custom Applications" + hanzi: "定制" # Means "custom-made" or "tailored". + description: "Application roles an inventory pulls from their own repositories into roles/; core never tracks them." + icon: "fas fa-puzzle-piece" + run_after: [web-app] + opt: title: "Webserver Optimation" hanzi: "优化" # Means "optimization". diff --git a/scripts/lint/ansible.sh b/scripts/lint/ansible.sh index 59ffced477..e647486e9e 100755 --- a/scripts/lint/ansible.sh +++ b/scripts/lint/ansible.sh @@ -46,5 +46,9 @@ if lint_diff_active ansible; then fi ansible-lint +mapfile -t custom_roles < <(lint_custom_roles) +if [[ "${#custom_roles[@]}" -gt 0 ]]; then + ansible-lint "${custom_roles[@]}" +fi ansible_lint_stamp_record "${current}" diff --git a/scripts/lint/autoformat.sh b/scripts/lint/autoformat.sh index e5af120034..6d8107eba4 100755 --- a/scripts/lint/autoformat.sh +++ b/scripts/lint/autoformat.sh @@ -47,8 +47,10 @@ run_ruff() { return 0 fi local rc=0 - ruff format . || rc=$? - [[ $rc -eq 0 ]] && { ruff check . --fix || rc=$?; } + local -a custom_roles=() + mapfile -t custom_roles < <(lint_custom_roles) + ruff format . "${custom_roles[@]}" || rc=$? + [[ $rc -eq 0 ]] && { ruff check . --fix "${custom_roles[@]}" || rc=$?; } if [[ $rc -eq 0 ]]; then write_status ruff OK else @@ -139,17 +141,22 @@ run_ansible_lint() { return 0 fi local rc=0 out yaml_before - local -a targets=() + local -a targets=() custom_roles=() if lint_diff_active ansible; then mapfile -t targets < <(lint_diff_files '\.ya?ml$' | ansible_lint_included) if [[ "${#targets[@]}" -eq 0 ]]; then write_status ansible-lint OK return 0 fi + else + mapfile -t custom_roles < <(lint_custom_roles) fi out="$(mktemp)" yaml_before="$(ansible_lint_fingerprint yaml)" ansible-lint --fix "${targets[@]}" >"${out}" 2>&1 || rc=$? + if [[ "${#custom_roles[@]}" -gt 0 && ($rc -eq 0 || $rc -eq 2 || $rc -eq 8) ]]; then + ansible-lint --fix "${custom_roles[@]}" >>"${out}" 2>&1 || rc=$? + fi if [[ $rc -eq 0 && "${#targets[@]}" -eq 0 && "$(ansible_lint_fingerprint yaml)" == "${yaml_before}" ]]; then touch "${status_dir}/.ansible-lint-clean" fi diff --git a/scripts/lint/diff/files.sh b/scripts/lint/diff/files.sh index 9880b42a9f..3fb0288d2d 100755 --- a/scripts/lint/diff/files.sh +++ b/scripts/lint/diff/files.sh @@ -1,7 +1,12 @@ # shellcheck shell=bash -# Param lint_diff_write : on the host, writes build/lint-diff-.txt with every file changed since the merge-base of HEAD and INFINITO_LINT_DIFF_BASE (untracked files included) and prints its path; prints nothing when every file must be linted. +# Param lint_diff_write : on the host, writes build/lint-diff-.txt with every file changed since the merge-base of HEAD and INFINITO_LINT_DIFF_BASE (untracked files and pulled custom roles included) and prints its path; prints nothing when every file must be linted. # Param lint_diff_active : succeeds when LINT_DIFF_FILES names a list and none of its files is configuration of that tool or a shared lint script. # Param lint_diff_files : prints the listed files matching that still exist, one per line. +# Param lint_custom_roles: prints every pulled custom role directory (roles/web-app-cst-*), one per line; Git ignores them, so tools that honour .gitignore get them passed explicitly. + +lint_custom_roles() { + find roles -mindepth 1 -maxdepth 1 -type d -name 'web-app-cst-*' -print | LC_ALL=C sort +} lint_diff_write() { local list="build/lint-diff-$1.txt" merge_base @@ -16,6 +21,7 @@ lint_diff_write() { { git diff --name-only --diff-filter=d "${merge_base}" git ls-files --others --exclude-standard + lint_custom_roles | xargs -r -d '\n' -I{} find {} -type f -print } | LC_ALL=C sort -u >"${list}" echo "lint $1: $(wc -l <"${list}") file(s) changed since ${INFINITO_LINT_DIFF_BASE}; ansible, javascript, markdown, mermaid, shellcheck and sql check only those unless their configuration changed" >&2 printf '%s\n' "${list}" diff --git a/scripts/lint/python.sh b/scripts/lint/python.sh index c5da2efda9..420b4d60ff 100755 --- a/scripts/lint/python.sh +++ b/scripts/lint/python.sh @@ -11,6 +11,10 @@ cd "${REPO_ROOT}" RUFF_CACHE_DIR="build/ruff-cache-$(id -u)" export RUFF_CACHE_DIR +# shellcheck source=scripts/lint/diff/files.sh +source "${SCRIPT_DIR}/diff/files.sh" +mapfile -t custom_roles < <(lint_custom_roles) + shfmt -w scripts -ruff format . -ruff check . --fix +ruff format . "${custom_roles[@]}" +ruff check . --fix "${custom_roles[@]}" diff --git a/scripts/tests/deploy/act/copy_worktree_to_node.sh b/scripts/tests/deploy/act/copy_worktree_to_node.sh index a81a00db0f..e72b490174 100755 --- a/scripts/tests/deploy/act/copy_worktree_to_node.sh +++ b/scripts/tests/deploy/act/copy_worktree_to_node.sh @@ -14,6 +14,9 @@ cd "${repo_root}" # shellcheck source=scripts/meta/env/load.sh source "${repo_root}/scripts/meta/env/load.sh" -git ls-files -z -m -o --exclude-standard | +{ + git ls-files -z -m -o --exclude-standard + git ls-files -z -o -i --exclude-standard -- 'roles/web-app-cst-*' +} | tar --null -cf - -T - | docker exec -i "${node}" tar -C "${INFINITO_NODE_SRC_DIR:?}" -xpf - diff --git a/scripts/tests/deploy/act/workflow.sh b/scripts/tests/deploy/act/workflow.sh index 2f39f599ae..0d190fffa1 100755 --- a/scripts/tests/deploy/act/workflow.sh +++ b/scripts/tests/deploy/act/workflow.sh @@ -54,7 +54,8 @@ else fi cmd=(act "${ACT_EVENT}" -W "${_act_workflow}") -_act_workflows_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)/.github/workflows" +_act_repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +_act_workflows_dir="${_act_repo_root}/.github/workflows" while read -r _act_label; do cmd+=(-P "${_act_label}=${ACT_PLATFORM_IMAGE}") done < <(grep -rhE 'runs-on:' "${_act_workflows_dir}" | grep -oE 'ubuntu-[0-9]+\.[0-9]+' | sort -u) @@ -99,6 +100,12 @@ if [[ "${ACT_RM}" == "true" ]]; then fi cmd+=(--cache-server-addr 127.0.0.1) cmd+=(--artifact-server-addr 127.0.0.1) +if [[ "${ACT_BIND}" != "true" ]] && compgen -G "${_act_repo_root}/roles/web-app-cst-*" >/dev/null; then + # Exception: act copies the checkout through .gitignore, which drops the + # pulled custom roles, so the job binds the checkout to keep them. + echo "act: binding the checkout to carry the pulled custom roles (roles/web-app-cst-*)" + ACT_BIND=true +fi if [[ "${ACT_BIND}" == "true" ]]; then cmd+=(--bind) fi diff --git a/scripts/tests/deploy/local/deploy/container/initialize/all.sh b/scripts/tests/deploy/local/deploy/container/initialize/all.sh index e16dd4ca83..77172e72df 100755 --- a/scripts/tests/deploy/local/deploy/container/initialize/all.sh +++ b/scripts/tests/deploy/local/deploy/container/initialize/all.sh @@ -56,6 +56,7 @@ echo ">>> Deploying against ${INFINITO_INVENTORY_FILE}" infinito administration deploy dedicated "${INFINITO_INVENTORY_FILE}" \ --skip-backup \ + --skip-pull \ --debug \ --log "${INFINITO_SRC_DIR}/logs" \ -l "${INFINITO_LIMIT_HOST}" \ diff --git a/scripts/tests/deploy/local/deploy/container/update/all.sh b/scripts/tests/deploy/local/deploy/container/update/all.sh index cc680128ae..6ea35f8fcc 100755 --- a/scripts/tests/deploy/local/deploy/container/update/all.sh +++ b/scripts/tests/deploy/local/deploy/container/update/all.sh @@ -27,6 +27,7 @@ echo ">>> entry.sh bootstrap" cmd=(infinito administration deploy dedicated "${INFINITO_INVENTORY_FILE}" --skip-backup --skip-cleanup + --skip-pull -l "${INFINITO_LIMIT_HOST}" --diff -vv diff --git a/scripts/tests/deploy/local/deploy/container/update/selection.sh b/scripts/tests/deploy/local/deploy/container/update/selection.sh index b5f82f0ad2..70591978de 100755 --- a/scripts/tests/deploy/local/deploy/container/update/selection.sh +++ b/scripts/tests/deploy/local/deploy/container/update/selection.sh @@ -53,6 +53,7 @@ read -ra app_ids <<<"${apps}" cmd=(infinito administration deploy dedicated "${INFINITO_INVENTORY_FILE}" --skip-backup --skip-cleanup + --skip-pull --id "${app_ids[@]}" -l localhost --diff diff --git a/tests/integration/infrastructure/compose/test_no_raw_docker.py b/tests/integration/infrastructure/compose/test_no_raw_docker.py index 1357d62f5b..0865cda7ee 100644 --- a/tests/integration/infrastructure/compose/test_no_raw_docker.py +++ b/tests/integration/infrastructure/compose/test_no_raw_docker.py @@ -1,14 +1,13 @@ from __future__ import annotations import re -import subprocess import unittest from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING from utils.annotations.suppress import is_suppressed_at, is_suppressed_in_head -from utils.cache.files import iter_project_files, read_text +from utils.cache.files import iter_project_files, read_text, repository_files from . import PROJECT_ROOT @@ -162,12 +161,7 @@ RULES: tuple[tuple[str, re.Pattern, str], ...] = ( def git_ls_files(root: Path) -> list[Path]: try: - out = subprocess.check_output( - ["git", "-C", str(root), "ls-files", "-z"], - stderr=subprocess.STDOUT, - ) - rels = [p for p in out.decode("utf-8", errors="replace").split("\0") if p] - return [root / p for p in rels] + return [root / p for p in repository_files(untracked=True)] except Exception: results: list[Path] = [] for path_str in iter_project_files(): diff --git a/tests/integration/roles/custom/README.md b/tests/integration/roles/custom/README.md new file mode 100644 index 0000000000..c785435a5a --- /dev/null +++ b/tests/integration/roles/custom/README.md @@ -0,0 +1,5 @@ +# Custom Role Tests 🧩 + +Integration tests that pull a custom role (`web-app-cst-*`) into a throwaway copy of the core checkout with its real `.gitignore` and check that Git never sees it. + +For framework and `make test-integration` usage see [integration.md](../../../../docs/contributing/actions/testing/integration.md). diff --git a/tests/integration/roles/custom/__init__.py b/tests/integration/roles/custom/__init__.py new file mode 100644 index 0000000000..039b52c789 --- /dev/null +++ b/tests/integration/roles/custom/__init__.py @@ -0,0 +1,3 @@ +from tests.integration.roles import PROJECT_ROOT + +__all__ = ["PROJECT_ROOT"] diff --git a/tests/integration/roles/custom/test_git_status.py b/tests/integration/roles/custom/test_git_status.py new file mode 100644 index 0000000000..c713840a00 --- /dev/null +++ b/tests/integration/roles/custom/test_git_status.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +from cli.administration.roles.placement import clean, pull +from utils.cache.files import read_text +from utils.roles.mapping import ROLE_FILE_META_MAIN, ROLE_FILE_VARS_MAIN + +from . import PROJECT_ROOT + +ROLE = "web-app-cst-demo" +_ENV = { + **{key: value for key, value in os.environ.items() if not key.startswith("GIT_")}, + "GIT_CONFIG_GLOBAL": os.devnull, +} + + +def _git(repository: Path, *arguments: str) -> str: + return subprocess.run( + [ + "git", + "-c", + "user.name=ci", + "-c", + "user.email=ci@example.invalid", + "-c", + "commit.gpgsign=false", + *arguments, + ], + cwd=repository, + env=_ENV, + check=True, + capture_output=True, + text=True, + ).stdout + + +def _write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +@unittest.skipUnless(shutil.which("git"), "git is required for this test") +class TestPulledRoleStaysOutOfGit(unittest.TestCase): + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + base = Path(tmp.name) + self.checkout = base / "core" + self.roles = self.checkout / "roles" + self.work = self.checkout / "build" / "custom-roles" + _write( + self.checkout / ".gitignore", read_text(str(PROJECT_ROOT / ".gitignore")) + ) + _write( + self.roles / "web-app-nextcloud" / ROLE_FILE_META_MAIN, "galaxy_info: {}\n" + ) + _git(self.checkout, "init", "--quiet") + _git(self.checkout, "add", "--all") + _git(self.checkout, "commit", "--quiet", "--no-verify", "--message", "core") + source = base / "landingpage" / "ansible" / "roles" / ROLE + _write(source / ROLE_FILE_META_MAIN, "galaxy_info: {}\n") + _write(source / ROLE_FILE_VARS_MAIN, f"application_id: {ROLE}\n") + applications = { + ROLE: { + "custom": True, + "source": { + "url": str(base / "landingpage"), + "path": f"ansible/roles/{ROLE}", + }, + } + } + _write( + base / "inventory" / "host_vars" / "localhost.yml", + json.dumps({"applications": applications}), + ) + _write( + base / "inventory" / "devices.yml", + json.dumps({"all": {"children": {ROLE: {"hosts": {"localhost": {}}}}}}), + ) + self.inventory = base / "inventory" / "devices.yml" + + def test_a_pulled_role_leaves_git_status_empty(self) -> None: + pull(self.inventory, self.roles, self.work) + self.assertTrue((self.roles / ROLE / ROLE_FILE_VARS_MAIN).is_file()) + self.assertEqual("", _git(self.checkout, "status", "--porcelain")) + self.assertIn( + f"roles/{ROLE}/", + _git(self.checkout, "status", "--porcelain", "--ignored"), + ) + + def test_clean_leaves_only_the_tracked_roles(self) -> None: + pull(self.inventory, self.roles, self.work) + clean(self.roles, self.work) + self.assertEqual(["web-app-nextcloud"], [e.name for e in self.roles.iterdir()]) + self.assertEqual( + "", _git(self.checkout, "status", "--porcelain", "--ignored", "roles") + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/roles/naming/test_custom_prefix_reserved.py b/tests/integration/roles/naming/test_custom_prefix_reserved.py new file mode 100644 index 0000000000..6277fc0ee5 --- /dev/null +++ b/tests/integration/roles/naming/test_custom_prefix_reserved.py @@ -0,0 +1,38 @@ +"""The custom-role prefix is reserved for roles an inventory pulls. + +``roles/web-app-cst-*`` is where the pull places the custom roles an +inventory declares, and Git ignores that path. A role committed to core +under that prefix would be replaced or deleted by the next pull, so no +tracked file may live there. +""" + +from __future__ import annotations + +import subprocess +import unittest + +from utils.roles.custom import PREFIX, ROLES_GLOB + +from . import PROJECT_ROOT + + +class TestCustomPrefixReserved(unittest.TestCase): + def test_no_in_tree_role_uses_the_custom_prefix(self): + tracked = subprocess.run( + ["git", "-c", "safe.directory=*", "ls-files", "-z", "--", ROLES_GLOB], + cwd=PROJECT_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout + roles = sorted({path.split("/")[1] for path in tracked.split("\0") if path}) + self.assertEqual( + [], + roles, + f"In-tree roles must not start with {PREFIX}, the prefix of the custom " + f"roles an inventory pulls into roles/; rename: {', '.join(roles)}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/lint/ansible/roles/test_readme.py b/tests/lint/ansible/roles/test_readme.py index 039f17be3a..7fb93e3e35 100644 --- a/tests/lint/ansible/roles/test_readme.py +++ b/tests/lint/ansible/roles/test_readme.py @@ -22,7 +22,8 @@ Enforced (hard fail) rules single point of truth ``galaxy_info.author`` read from the role's own ``meta/main.yml``. The README author (bold, optionally wrapped in a Markdown link) MUST equal that value byte-for-byte, so a role's - metadata and its README stay in sync. + metadata and its README stay in sync. A custom role (``web-app-cst-*``) + states its own licence, so the license sentence is not demanded there. 5. **Cosmos diagram.** Every role README MUST carry a ``## Cosmos`` H2 whose body contains a ```mermaid fenced diagram placing the role in the Infinito.Nexus cosmos (capabilities / dependencies / cosmos). The @@ -57,6 +58,7 @@ from cli.build.docs.readme.generate import _app_name, _managed_blocks from cli.build.docs.readme.sections import parse_readme from utils.cache.files import PROJECT_ROOT, read_text from utils.cache.yaml import load_yaml_any +from utils.roles.custom import is_custom from utils.roles.mapping import ROLE_FILE_META_MAIN from utils.roles.validation.invokable import _get_invokable_paths, _is_role_invokable @@ -111,14 +113,28 @@ _CREDITS_IMPL_RE = re.compile( r"\*\*\." ) -_CREDITS_TAIL: str = ( +_CREDITS_PROJECT: str = ( "Part of the [Infinito.Nexus Project](https://s.infinito.nexus/code) " "and maintained by [Kevin Veen-Birkenbach](https://www.veen.world).\n" +) +_CREDITS_LICENSE: str = ( "Licensed under the [Infinito.Nexus Community License (Non-Commercial)]" "(https://s.infinito.nexus/license).\n" ) +def _credits_tail(role_name: str) -> str: + """Return the canonical Credits tail one role README MUST carry. + + Args: + role_name: the role directory name; a custom role states its own + licence, so its tail stops before the Infinito.Nexus licence. + """ + if is_custom(role_name): + return _CREDITS_PROJECT + return _CREDITS_PROJECT + _CREDITS_LICENSE + + def _role_readmes() -> list[Path]: if not ROLES_DIR.is_dir(): return [] @@ -221,7 +237,7 @@ def _validate_credits(text: str, path: Path) -> list[str]: f"{author!r} in meta/main.yml (the author SPOT)" ) - if _CREDITS_TAIL not in text: + if _credits_tail(path.parent.name) not in text: problems.append( "Credits tail does not match the canonical wording " "(see docs/contributing/artefact/files/role/readme_md.md)" diff --git a/tests/lint/ansible/services/test_memory_parameters_use_lookup.py b/tests/lint/ansible/services/test_memory_parameters_use_lookup.py index 70a725928d..0eb0410909 100644 --- a/tests/lint/ansible/services/test_memory_parameters_use_lookup.py +++ b/tests/lint/ansible/services/test_memory_parameters_use_lookup.py @@ -33,14 +33,13 @@ written another way is not seen; add its spelling when a role introduces one. from __future__ import annotations import re -import subprocess import unittest from bisect import bisect_right from collections import defaultdict from functools import cache from typing import TYPE_CHECKING, ClassVar -from utils.cache.files import read_text +from utils.cache.files import read_text, repository_files from utils.roles.mapping import ROLE_FILE_TEMPL_COMPOSE from . import PROJECT_ROOT @@ -278,22 +277,9 @@ def _scope(rel: str) -> str: @cache def _tracked_texts() -> dict[str, str]: - out = subprocess.check_output( - [ - "git", - "-c", - "safe.directory=*", - "-C", - str(PROJECT_ROOT), - "ls-files", - "roles", - _SHARED_SCOPE, - ], - text=True, - ) texts: dict[str, str] = {} - for rel in out.splitlines(): - if rel.endswith(".md"): + for rel in repository_files(): + if not rel.startswith(("roles/", f"{_SHARED_SCOPE}/")) or rel.endswith(".md"): continue try: texts[rel] = read_text(str(PROJECT_ROOT / rel)) diff --git a/tests/lint/filesystem/env/test_env_file_schema.py b/tests/lint/filesystem/env/test_env_file_schema.py index 581ca86338..d96db02ce9 100644 --- a/tests/lint/filesystem/env/test_env_file_schema.py +++ b/tests/lint/filesystem/env/test_env_file_schema.py @@ -48,12 +48,11 @@ Style guidance for the one comment line (see `docs/contributing/documentation.md from __future__ import annotations import re -import subprocess import unittest from dataclasses import dataclass from typing import TYPE_CHECKING -from utils.cache.files import read_text +from utils.cache.files import read_text, repository_files from . import PROJECT_ROOT @@ -257,25 +256,10 @@ def validate_env_file(path: Path, *, strict: bool) -> list[Violation]: return violations -def _git_ls_files() -> list[str]: - out = subprocess.check_output( - [ - "git", - "-c", - "safe.directory=*", - "-C", - str(PROJECT_ROOT), - "ls-files", - ], - text=True, - ) - return [line for line in out.splitlines() if line] - - def _scan_targets() -> list[tuple[Path, bool]]: """Return (path, strict) pairs for every env-file the lint covers.""" targets: list[tuple[Path, bool]] = [] - for rel in _git_ls_files(): + for rel in repository_files(): if rel.endswith(".env"): strict = rel.startswith("env/") or rel == ".env" targets.append((PROJECT_ROOT / rel, strict)) diff --git a/tests/lint/filesystem/playwright/test_onion_request_timeout.py b/tests/lint/filesystem/playwright/test_onion_request_timeout.py index c2677c4b81..b62a5841b2 100644 --- a/tests/lint/filesystem/playwright/test_onion_request_timeout.py +++ b/tests/lint/filesystem/playwright/test_onion_request_timeout.py @@ -28,12 +28,11 @@ never an onion host (localhost, an external clearnet API, ...). from __future__ import annotations import re -import subprocess import unittest from dataclasses import dataclass from typing import TYPE_CHECKING -from utils.cache.files import read_text +from utils.cache.files import read_text, repository_files from . import PROJECT_ROOT @@ -55,14 +54,6 @@ class Violation: detail: str -def _git_ls_files() -> list[str]: - out = subprocess.check_output( - ["git", "-c", "safe.directory=*", "-C", str(PROJECT_ROOT), "ls-files"], - text=True, - ) - return [line for line in out.splitlines() if line] - - def _call_end(text: str, open_idx: int) -> int: """Index of the ``)`` matching the ``(`` at ``open_idx`` (string-aware). @@ -139,7 +130,7 @@ def _scan_file(path: Path) -> list[Violation]: def _scan_targets() -> list[Path]: return [ PROJECT_ROOT / rel - for rel in _git_ls_files() + for rel in repository_files() if rel.endswith(".js") and "/files/playwright/" in rel ] diff --git a/tests/lint/filesystem/python/test_python_shebangs.py b/tests/lint/filesystem/python/test_python_shebangs.py index e3143a9b77..3613a20bbd 100755 --- a/tests/lint/filesystem/python/test_python_shebangs.py +++ b/tests/lint/filesystem/python/test_python_shebangs.py @@ -2,11 +2,10 @@ from __future__ import annotations -import subprocess import unittest from pathlib import Path -from utils.cache.files import iter_project_files +from utils.cache.files import iter_project_files, repository_files from . import PROJECT_ROOT @@ -21,14 +20,7 @@ class TestPythonShebangs(unittest.TestCase): def _tracked_files(self) -> list[Path]: try: - out = subprocess.check_output( - ["git", "-C", str(self.repo_root), "ls-files", "-z"], - stderr=subprocess.STDOUT, - ) - rel_paths = [ - p for p in out.decode("utf-8", errors="replace").split("\0") if p - ] - return [self.repo_root / p for p in rel_paths] + return [self.repo_root / p for p in repository_files(untracked=True)] except Exception: return [Path(p) for p in iter_project_files()] diff --git a/tests/lint/filesystem/shell/test_no_infinito_defaults.py b/tests/lint/filesystem/shell/test_no_infinito_defaults.py index 78f422afe0..4f06db4950 100644 --- a/tests/lint/filesystem/shell/test_no_infinito_defaults.py +++ b/tests/lint/filesystem/shell/test_no_infinito_defaults.py @@ -27,12 +27,11 @@ overrides not exposed as a stack-wide variable). from __future__ import annotations import re -import subprocess import unittest from dataclasses import dataclass from typing import TYPE_CHECKING -from utils.cache.files import read_text +from utils.cache.files import read_text, repository_files from . import PROJECT_ROOT @@ -56,21 +55,6 @@ class Violation: detail: str -def _git_ls_files() -> list[str]: - out = subprocess.check_output( - [ - "git", - "-c", - "safe.directory=*", - "-C", - str(PROJECT_ROOT), - "ls-files", - ], - text=True, - ) - return [line for line in out.splitlines() if line] - - def _scan_file(path: Path) -> list[Violation]: violations: list[Violation] = [] rel = path.relative_to(PROJECT_ROOT).as_posix() @@ -117,7 +101,7 @@ def _scan_file(path: Path) -> list[Violation]: def _scan_targets() -> list[Path]: - return [PROJECT_ROOT / rel for rel in _git_ls_files() if rel.endswith(".sh")] + return [PROJECT_ROOT / rel for rel in repository_files() if rel.endswith(".sh")] class TestShellNoInfinitoDefaults(unittest.TestCase): diff --git a/tests/lint/filesystem/shell/test_no_or_true.py b/tests/lint/filesystem/shell/test_no_or_true.py index f414bb593c..07c7da8610 100644 --- a/tests/lint/filesystem/shell/test_no_or_true.py +++ b/tests/lint/filesystem/shell/test_no_or_true.py @@ -17,12 +17,11 @@ exists to stop *new* blanket ``|| true`` from creeping in. from __future__ import annotations import re -import subprocess import unittest from dataclasses import dataclass from typing import TYPE_CHECKING -from utils.cache.files import read_text +from utils.cache.files import read_text, repository_files from . import PROJECT_ROOT @@ -40,21 +39,6 @@ class Violation: detail: str -def _git_ls_files() -> list[str]: - out = subprocess.check_output( - [ - "git", - "-c", - "safe.directory=*", - "-C", - str(PROJECT_ROOT), - "ls-files", - ], - text=True, - ) - return [line for line in out.splitlines() if line] - - def _scan_file(path: Path) -> list[Violation]: violations: list[Violation] = [] rel = path.relative_to(PROJECT_ROOT).as_posix() @@ -81,7 +65,7 @@ def _scan_file(path: Path) -> list[Violation]: def _scan_targets() -> list[Path]: - return [PROJECT_ROOT / rel for rel in _git_ls_files() if rel.endswith(".sh")] + return [PROJECT_ROOT / rel for rel in repository_files() if rel.endswith(".sh")] class TestShellNoOrTrue(unittest.TestCase): diff --git a/tests/lint/filesystem/shell/test_no_path_defaults.py b/tests/lint/filesystem/shell/test_no_path_defaults.py index 4d050a0c92..c4253fa693 100644 --- a/tests/lint/filesystem/shell/test_no_path_defaults.py +++ b/tests/lint/filesystem/shell/test_no_path_defaults.py @@ -30,12 +30,11 @@ only when the shell context genuinely cannot consume ``default.env``. from __future__ import annotations import re -import subprocess import unittest from dataclasses import dataclass from typing import TYPE_CHECKING -from utils.cache.files import read_text +from utils.cache.files import read_text, repository_files from . import PROJECT_ROOT @@ -56,24 +55,6 @@ class Violation: detail: str -def _git_ls_files() -> list[str]: - """List tracked files; ``safe.directory=*`` bypasses git's ownership - check, which fails inside the dev container when the bind-mounted - repo's UID does not match the container user.""" - out = subprocess.check_output( - [ - "git", - "-c", - "safe.directory=*", - "-C", - str(PROJECT_ROOT), - "ls-files", - ], - text=True, - ) - return [line for line in out.splitlines() if line] - - def _scan_file(path: Path) -> list[Violation]: violations: list[Violation] = [] rel = path.relative_to(PROJECT_ROOT).as_posix() @@ -104,7 +85,7 @@ def _scan_file(path: Path) -> list[Violation]: def _scan_targets() -> list[Path]: - return [PROJECT_ROOT / rel for rel in _git_ls_files() if rel.endswith(".sh")] + return [PROJECT_ROOT / rel for rel in repository_files() if rel.endswith(".sh")] class TestShellNoPathDefaults(unittest.TestCase): diff --git a/tests/lint/filesystem/test_no_rsync_backup_flag.py b/tests/lint/filesystem/test_no_rsync_backup_flag.py index a2b47e659f..5361f106fa 100644 --- a/tests/lint/filesystem/test_no_rsync_backup_flag.py +++ b/tests/lint/filesystem/test_no_rsync_backup_flag.py @@ -18,12 +18,11 @@ Suppress on a per-line basis with a same-line ``# nocheck: rsync-backup -- from __future__ import annotations import re -import subprocess import unittest from dataclasses import dataclass from pathlib import Path -from utils.cache.files import read_text +from utils.cache.files import read_text, repository_files from . import PROJECT_ROOT @@ -44,14 +43,6 @@ class Violation: detail: str -def _git_ls_files() -> list[str]: - out = subprocess.check_output( - ["git", "-c", "safe.directory=*", "-C", str(PROJECT_ROOT), "ls-files"], - text=True, - ) - return [line for line in out.splitlines() if line] - - def _offending_flag(line: str) -> str | None: """Name the backup flag on a line, or None when it carries none. @@ -101,7 +92,7 @@ def _scan_file(path: Path) -> list[Violation]: def _scan_targets() -> list[Path]: return [ PROJECT_ROOT / rel - for rel in _git_ls_files() + for rel in repository_files() if rel.endswith(_SCANNED_SUFFIXES) and rel != _SELF_REL ] diff --git a/tests/lint/repository/documentation/test_todo_transparency.py b/tests/lint/repository/documentation/test_todo_transparency.py index 14e8157cf4..edc9df2c00 100644 --- a/tests/lint/repository/documentation/test_todo_transparency.py +++ b/tests/lint/repository/documentation/test_todo_transparency.py @@ -9,13 +9,12 @@ readable. from __future__ import annotations import re -import subprocess import unittest from dataclasses import dataclass from pathlib import Path from utils.annotations.message import in_github_actions, warning -from utils.cache.files import iter_project_files, read_text +from utils.cache.files import iter_project_files, read_text, repository_files from . import PROJECT_ROOT @@ -79,12 +78,7 @@ class TodoFinding: def tracked_files(root: Path) -> list[Path]: try: - out = subprocess.check_output( - ["git", "-C", str(root), "ls-files", "-z"], - stderr=subprocess.STDOUT, - ) - rel_paths = [p for p in out.decode("utf-8", errors="replace").split("\0") if p] - return [root / rel for rel in rel_paths] + return [root / rel for rel in repository_files(untracked=True)] except Exception: return [Path(p) for p in iter_project_files()] diff --git a/tests/lint/repository/test_comments_valid.py b/tests/lint/repository/test_comments_valid.py index 9158285e18..68d417da69 100644 --- a/tests/lint/repository/test_comments_valid.py +++ b/tests/lint/repository/test_comments_valid.py @@ -40,9 +40,9 @@ intentionally OUT of scope (reliably distinguishing them from ``#`` inside a string needs a full lexer per language, and false positives would make the linter unusable). -SCOPE: every tracked file of a checked type, plus untracked new ones. The -policy holds for the whole repository, not only for the files a change happens -to touch. +SCOPE: every tracked file of a checked type, plus untracked new ones and the +custom roles pulled into ``roles/``. The policy holds for the whole repository, +not only for the files a change happens to touch. A whole file opts out with a ``# nocheck: comments-valid`` marker in its first 30 lines (any comment prefix, e.g. ``{# nocheck: comments-valid #}`` for Jinja), @@ -60,11 +60,12 @@ import unittest from typing import TYPE_CHECKING from utils.annotations.suppress import is_suppressed_in_head -from utils.cache.files import read_text +from utils.cache.files import read_text, repository_files, untracked_files from . import PROJECT_ROOT if TYPE_CHECKING: + from collections.abc import Callable from pathlib import Path _FILE_NOCHECK_RULE = "comments-valid" @@ -450,32 +451,20 @@ def find_invalid_comments(path: Path): _EXTS = (".py", ".sh", ".yml", ".yaml", ".rb", ".php", ".css", ".j2") -def _git_lines(args: list[str]) -> list[str]: - """Run a git command and return its stdout lines. - - safe.directory='*' is required because the make-test container mounts the - repo as a different uid; bare git would abort with dubious-ownership and the - linter would silently scan zero files. - """ +def _listed(listing: Callable[[], tuple[str, ...]]) -> set[str]: try: - result = subprocess.run( - ["git", "-c", "safe.directory=*", *args], - cwd=str(PROJECT_ROOT), - capture_output=True, - text=True, - check=True, - ) + return set(listing()) except (OSError, subprocess.SubprocessError): - return [] - return result.stdout.splitlines() + return set() def _all_targets(): - """Every tracked file of a checked type, plus untracked new ones.""" - rel = set(_git_lines(["ls-files"])) - rel |= set(_git_lines(["ls-files", "--others", "--exclude-standard"])) + """Every tracked file of a checked type, plus untracked new ones. + + A failing git listing loses only its own files. + """ targets = [] - for r in sorted(rel): + for r in sorted(_listed(repository_files) | _listed(untracked_files)): if r.endswith(_EXTS) or r.rsplit("/", 1)[-1] == "Dockerfile": path = PROJECT_ROOT / r if path.is_file(): diff --git a/tests/lint/repository/test_no_debian_codename_tags.py b/tests/lint/repository/test_no_debian_codename_tags.py index 5445e3ab86..7a0634813d 100644 --- a/tests/lint/repository/test_no_debian_codename_tags.py +++ b/tests/lint/repository/test_no_debian_codename_tags.py @@ -20,7 +20,7 @@ the way every other version pin does. Scope ----- -Every tracked file. Two shapes are recognised: +Every tracked file and every pulled custom role. Two shapes are recognised: * a ``name:tag`` image reference with no space after the colon, which covers ``image:`` values, Dockerfile ``FROM`` lines and heredocs, @@ -46,13 +46,12 @@ it buys can never be raised. from __future__ import annotations import re -import subprocess import unittest from dataclasses import dataclass from typing import TYPE_CHECKING from utils.annotations.suppress import is_suppressed_at -from utils.cache.files import read_text +from utils.cache.files import read_text, repository_files from . import PROJECT_ROOT @@ -145,23 +144,13 @@ def _references_on(line: str) -> list[tuple[str, str]]: return found -def _git_lines(*args: str) -> list[str]: - out = subprocess.check_output( - ["git", "-c", "safe.directory=*", "-C", str(PROJECT_ROOT), *args], - text=True, - ) - return [line for line in out.splitlines() if line] - - def _targets() -> set[str]: """Every tracked file plus every untracked new one. A role added but not yet staged carries its pins like any other, so the lint MUST see it before the commit rather than after. """ - return set(_git_lines("ls-files")) | set( - _git_lines("ls-files", "--others", "--exclude-standard") - ) + return set(repository_files(untracked=True)) def _scan_file(path: Path) -> list[Violation]: diff --git a/tests/unit/python/cli/administration/deploy/dedicated/test_command.py b/tests/unit/python/cli/administration/deploy/dedicated/test_command.py index df0d5b1249..9476110fbd 100644 --- a/tests/unit/python/cli/administration/deploy/dedicated/test_command.py +++ b/tests/unit/python/cli/administration/deploy/dedicated/test_command.py @@ -1,9 +1,43 @@ -# cli/administration/deploy/dedicated/tests/test_command_ids.py from __future__ import annotations import unittest +from types import SimpleNamespace +from unittest import mock -from cli.administration.deploy.dedicated.command import _normalize_app_ids +from cli.administration.deploy.dedicated import command +from cli.administration.deploy.dedicated.command import ( + _normalize_app_ids, + pull_custom_roles, +) + + +class TestPullCustomRoles(unittest.TestCase): + @mock.patch("cli.administration.deploy.dedicated.command.pull_or_exit") + def test_pulls_the_inventory_by_default(self, pull_or_exit): + pull_custom_roles(SimpleNamespace(inventory="inv/devices.yml", skip_pull=False)) + pull_or_exit.assert_called_once_with("inv/devices.yml") + + @mock.patch("cli.administration.deploy.dedicated.command.pull_or_exit") + def test_skip_pull_keeps_the_placed_roles(self, pull_or_exit): + pull_custom_roles(SimpleNamespace(inventory="inv/devices.yml", skip_pull=True)) + pull_or_exit.assert_not_called() + + def test_main_pulls_before_anything_reads_roles(self): + calls: list[str] = [] + args = SimpleNamespace(inventory="inv/devices.yml", skip_pull=False) + with ( + mock.patch.object(command, "parse_args", return_value=(args, [], {})), + mock.patch.object( + command, "pull_or_exit", side_effect=lambda _inv: calls.append("pull") + ), + mock.patch.object( + command, + "run_from_args", + side_effect=lambda *_a: calls.append("deploy") or 0, + ), + ): + self.assertEqual(0, command.main([])) + self.assertEqual(["pull", "deploy"], calls) class TestNormalizeAppIds(unittest.TestCase): diff --git a/tests/unit/python/cli/administration/deploy/dedicated/test_command_passthrough.py b/tests/unit/python/cli/administration/deploy/dedicated/test_command_passthrough.py index a86a187859..3dc5968010 100644 --- a/tests/unit/python/cli/administration/deploy/dedicated/test_command_passthrough.py +++ b/tests/unit/python/cli/administration/deploy/dedicated/test_command_passthrough.py @@ -7,6 +7,7 @@ from cli.administration.deploy.dedicated import command as dedicated_command class TestDedicatedCommandPassthrough(unittest.TestCase): + @unittest.mock.patch("cli.administration.deploy.dedicated.command.pull_or_exit") @unittest.mock.patch( "cli.administration.deploy.dedicated.command.validate_application_ids" ) @@ -29,6 +30,7 @@ class TestDedicatedCommandPassthrough(unittest.TestCase): mock_load_modes_from_yaml, mock_run_ansible_playbook, mock_validate_application_ids, + _mock_pull_or_exit, ): mock_load_modes_from_yaml.return_value = [] mock_add_dynamic_mode_args.return_value = {} diff --git a/tests/unit/python/cli/administration/deploy/dedicated/test_main.py b/tests/unit/python/cli/administration/deploy/dedicated/test_main.py index 054fd8aa82..d21e60e63d 100644 --- a/tests/unit/python/cli/administration/deploy/dedicated/test_main.py +++ b/tests/unit/python/cli/administration/deploy/dedicated/test_main.py @@ -15,6 +15,7 @@ class TestDedicatedMainParser(unittest.TestCase): self.assertIn("limit", opts) self.assertIn("password_file", opts) self.assertIn("skip_build", opts) + self.assertIn("skip_pull", opts) self.assertIn("id", opts) self.assertIn("verbose", opts) self.assertIn("diff", opts) diff --git a/tests/unit/python/cli/administration/deploy/development/deploy/__init__.py b/tests/unit/python/cli/administration/deploy/development/deploy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/cli/administration/deploy/development/deploy/test_run.py b/tests/unit/python/cli/administration/deploy/development/deploy/test_run.py new file mode 100644 index 0000000000..53110503bb --- /dev/null +++ b/tests/unit/python/cli/administration/deploy/development/deploy/test_run.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import unittest +from types import SimpleNamespace +from unittest import mock + +from cli.administration.deploy.development.deploy.run import _run_deploy + + +class TestRunDeploy(unittest.TestCase): + def deploy_cmd(self) -> list[str]: + compose = mock.Mock() + compose.exec.return_value = SimpleNamespace(returncode=0) + _run_deploy( + compose, + deploy_ids=["web-app-cst-demo"], + debug=False, + passthrough=[], + inventory_dir="/etc/inventories/dev", + container_name="infinito", + ) + return compose.exec.call_args.args[0] + + def test_deploys_the_generated_inventory(self): + cmd = self.deploy_cmd() + self.assertEqual("/etc/inventories/dev/devices.yml", cmd[3]) + self.assertEqual(["--id", "web-app-cst-demo"], cmd[-2:]) + + def test_generated_inventory_keeps_the_placed_custom_roles(self): + self.assertIn("--skip-pull", self.deploy_cmd()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/cli/administration/deploy/development/test_env_compose_file_args.py b/tests/unit/python/cli/administration/deploy/development/test_env_compose_file_args.py index e864110543..b269a9fca9 100644 --- a/tests/unit/python/cli/administration/deploy/development/test_env_compose_file_args.py +++ b/tests/unit/python/cli/administration/deploy/development/test_env_compose_file_args.py @@ -8,7 +8,9 @@ because an unsatisfied `:?` guard inside one of them aborts the whole stack. from __future__ import annotations import os +import tempfile import unittest +from pathlib import Path from unittest.mock import patch from cli.administration.deploy.development.env import compose_file_args @@ -23,6 +25,7 @@ _LOCAL_PRIMARY = { "INFINITO_CACHE_STACK": "", "INFINITO_GHCR_MIRROR": "", "INFINITO_PUBLISH_PORTS": "", + "INFINITO_EXTRA_MOUNTS": "", } _WORKTREE = { @@ -99,6 +102,35 @@ class TestComposeFileArgs(unittest.TestCase): compose_file_args()[-2:], ["-f", "compose/noports.override.yml"] ) + def test_extra_mounts_append_their_generated_override(self) -> None: + with ( + tempfile.TemporaryDirectory() as mount, + patch.dict( + os.environ, + {**_LOCAL_PRIMARY, "INFINITO_EXTRA_MOUNTS": mount}, + clear=False, + ), + patch( + "cli.administration.deploy.development.extra_mounts.write_mounts_override", + return_value="build/compose/extra-mounts.override.yml", + ) as write, + ): + args = compose_file_args() + self.assertEqual(write.call_args.args[1], [Path(mount)]) + self.assertEqual(args[-2:], ["-f", "build/compose/extra-mounts.override.yml"]) + + def test_an_entry_that_is_no_directory_stops_the_stack_naming_it(self) -> None: + with ( + patch.dict( + os.environ, + {**_LOCAL_PRIMARY, "INFINITO_EXTRA_MOUNTS": "/nonexistent/gui"}, + clear=False, + ), + self.assertRaises(SystemExit) as ctx, + ): + compose_file_args() + self.assertIn("/nonexistent/gui", str(ctx.exception.code)) + if __name__ == "__main__": # pragma: no cover unittest.main() diff --git a/tests/unit/python/cli/administration/deploy/development/test_extra_mounts.py b/tests/unit/python/cli/administration/deploy/development/test_extra_mounts.py new file mode 100644 index 0000000000..4f45d7165e --- /dev/null +++ b/tests/unit/python/cli/administration/deploy/development/test_extra_mounts.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from cli.administration.deploy.development.extra_mounts import ( + OVERRIDE, + parse_mounts, + write_mounts_override, +) +from utils.cache.yaml import load_yaml + + +class TestParseMounts(unittest.TestCase): + def test_existing_absolute_directories_are_returned(self) -> None: + with ( + tempfile.TemporaryDirectory() as first, + tempfile.TemporaryDirectory() as second, + ): + self.assertEqual( + parse_mounts(f"{first}, {second},"), [Path(first), Path(second)] + ) + + def test_empty_value_mounts_nothing(self) -> None: + self.assertEqual(parse_mounts(""), []) + + def test_relative_entry_is_rejected_naming_it(self) -> None: + with self.assertRaises(SystemExit) as ctx: + parse_mounts("relative/dir") + self.assertIn("relative/dir", str(ctx.exception.code)) + + def test_missing_directory_is_rejected_naming_it(self) -> None: + with tempfile.TemporaryDirectory() as tmp, self.assertRaises(SystemExit) as ctx: + parse_mounts(str(Path(tmp) / "absent")) + self.assertIn("absent", str(ctx.exception.code)) + + +class TestWriteMountsOverride(unittest.TestCase): + def test_each_directory_is_bound_read_only_at_its_own_path(self) -> None: + with tempfile.TemporaryDirectory() as repo: + relative = write_mounts_override(Path(repo), [Path("/srv/gui")]) + override = load_yaml(Path(repo) / relative) + self.assertEqual(relative, OVERRIDE) + self.assertEqual( + override, + { + "services": { + "infinito": { + "volumes": [ + { + "type": "bind", + "source": "/srv/gui", + "target": "/srv/gui", + "read_only": True, + } + ] + } + } + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/cli/administration/deploy/swarm/test_command.py b/tests/unit/python/cli/administration/deploy/swarm/test_command.py new file mode 100644 index 0000000000..ed6c33c2c5 --- /dev/null +++ b/tests/unit/python/cli/administration/deploy/swarm/test_command.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import unittest +from types import SimpleNamespace +from unittest import mock + +from cli.administration.deploy.swarm import command + + +class TestSwarmMain(unittest.TestCase): + def test_custom_roles_are_pulled_before_the_closure_reads_roles(self): + calls: list[str] = [] + args = SimpleNamespace(id=["web-app-cst-demo"], inventory="inv/devices.yml") + + def closure(ids, _inventory): + calls.append("closure") + return ids + + with ( + mock.patch.object(command, "parse_args", return_value=(args, [], {})), + mock.patch.object( + command, + "pull_custom_roles", + side_effect=lambda _args: calls.append("pull"), + ), + mock.patch.object(command, "swarm_deploy_targets", side_effect=closure), + mock.patch.object(command, "run_from_args", return_value=0) as run, + ): + self.assertEqual(0, command.main([])) + + self.assertEqual(["pull", "closure"], calls) + run.assert_called_once_with(args, [], {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/cli/administration/deploy/test_container.py b/tests/unit/python/cli/administration/deploy/test_container.py index 28bc83c29f..97c2bafed5 100644 --- a/tests/unit/python/cli/administration/deploy/test_container.py +++ b/tests/unit/python/cli/administration/deploy/test_container.py @@ -82,6 +82,35 @@ class TestEnsureImage(unittest.TestCase): ) +class TestRunInContainer(unittest.TestCase): + @unittest.mock.patch("subprocess.run") + @unittest.mock.patch( + "cli.administration.deploy.container.command.docker_exec", + return_value=subprocess.CompletedProcess(args=[], returncode=0), + ) + @unittest.mock.patch( + "cli.administration.deploy.container.command.start_ci_container", + return_value="infinito-ci-test", + ) + def test_generated_inventory_keeps_the_placed_custom_roles( + self, _start, mock_docker_exec, _run + ): + deploy_container.run_in_container( + image="infinito:latest", + build=False, + rebuild=False, + no_cache=False, + inventory_dir="/tmp/inventory", + inventory_args=[], + deploy_args=["--debug"], + ) + + deploy_cmd = mock_docker_exec.call_args_list[-1].args[1] + self.assertIn("cli.administration.deploy.dedicated", deploy_cmd) + self.assertIn("--skip-pull", deploy_cmd) + self.assertEqual("--debug", deploy_cmd[-1]) + + class TestMain(unittest.TestCase): @unittest.mock.patch("cli.administration.deploy.container.command.run_in_container") def test_main_requires_forwarded_args(self, mock_run_in_container): diff --git a/tests/unit/python/cli/administration/inventory/validate/test_applications.py b/tests/unit/python/cli/administration/inventory/validate/test_applications.py index aea6e30f54..3f9ed263c2 100644 --- a/tests/unit/python/cli/administration/inventory/validate/test_applications.py +++ b/tests/unit/python/cli/administration/inventory/validate/test_applications.py @@ -94,6 +94,24 @@ class TestCompareApplicationKeys(unittest.TestCase): self.assertEqual(len(result), 1) self.assertIn("services.dashbaord", result[0]) + def test_custom_role_declaration_keys_are_legal(self): + apps = { + "web-app-cst-demo": { + "custom": True, + "source": {"url": "https://x/y.git", "version": "v1", "path": "r"}, + "services": {"port": 8080}, + } + } + defaults = {"web-app-cst-demo": {"services": {"port": 8080}}} + self.assertEqual(compare_application_keys(apps, defaults, "src"), []) + + def test_declaration_keys_on_an_in_tree_role_are_flagged(self): + apps = {"app1": {"custom": True, "source": {"url": "https://x/y.git"}}} + defaults = {"app1": {"services": {}}} + result = compare_application_keys(apps, defaults, "src") + self.assertEqual(3, len(result)) + self.assertTrue(any(err.endswith("app1: custom") for err in result)) + if __name__ == "__main__": # pragma: no cover unittest.main() diff --git a/tests/unit/python/cli/administration/roles/__init__.py b/tests/unit/python/cli/administration/roles/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/cli/administration/roles/_fixtures.py b/tests/unit/python/cli/administration/roles/_fixtures.py new file mode 100644 index 0000000000..8b7302d911 --- /dev/null +++ b/tests/unit/python/cli/administration/roles/_fixtures.py @@ -0,0 +1,149 @@ +"""Throwaway role repositories, inventories and core roles/ trees for the pull tests.""" + +from __future__ import annotations + +import json +import os +import subprocess +from typing import TYPE_CHECKING + +from utils.roles.mapping import ( + ROLE_FILE_META_MAIN, + ROLE_FILE_TASKS_MAIN, + ROLE_FILE_VARS_MAIN, +) + +if TYPE_CHECKING: + from pathlib import Path + +ROLE = "web-app-cst-demo" +ROLE_PATH = f"ansible/roles/{ROLE}" + +_ENV = { + **{key: value for key, value in os.environ.items() if not key.startswith("GIT_")}, + "GIT_CONFIG_GLOBAL": os.devnull, +} + + +def git(cwd: Path, *args: str) -> str: + """Run git in *cwd* with a throwaway identity and no signing. + + Args: + cwd: the repository. + args: the git subcommand and its arguments. + """ + return subprocess.run( + [ + "git", + "-c", + "user.name=test", + "-c", + "user.email=test@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "tag.gpgsign=false", + *args, + ], + cwd=cwd, + env=_ENV, + capture_output=True, + text=True, + check=True, + ).stdout + + +def write(path: Path, text: str) -> Path: + """Write *text* to *path*, creating its parents. + + Args: + path: the file. + text: its content. + """ + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def role_repo(root: Path, application_id: str = ROLE) -> Path: + """A repository whose tag ``v1.0.0`` holds the role and whose branch ``feature`` moved on. + + Args: + root: directory the repository is created in. + application_id: the ``vars/main.yml`` application_id of the role. + """ + repo = root / "repo" + write(repo / "README.md", "repository root\n") + write(repo / ROLE_PATH / ROLE_FILE_META_MAIN, "galaxy_info: {}\n") + write(repo / ROLE_PATH / ROLE_FILE_VARS_MAIN, f"application_id: {application_id}\n") + write(repo / ROLE_PATH / ROLE_FILE_TASKS_MAIN, "release: v1\n") + git(repo, "init", "-q", "-b", "main") + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", "v1") + git(repo, "tag", "v1.0.0") + git(repo, "checkout", "-q", "-b", "feature") + write(repo / ROLE_PATH / ROLE_FILE_TASKS_MAIN, "release: feature\n") + git(repo, "commit", "-q", "-am", "feature") + return repo + + +def local_role(root: Path, group: str) -> Path: + """A checkout without Git history that holds the role *group* under ansible/roles/. + + Args: + root: directory the checkout is created in. + group: the role's group and application_id. + """ + checkout = root / f"{group}-checkout" + write(checkout / "ansible/roles" / group / ROLE_FILE_META_MAIN, "galaxy_info: {}\n") + write( + checkout / "ansible/roles" / group / ROLE_FILE_VARS_MAIN, + f"application_id: {group}\n", + ) + return checkout + + +def inventory(root: Path, groups: list[str], applications: dict) -> Path: + """An inventory whose host joins *groups* and whose host_vars set *applications*. + + Args: + root: directory the inventory is created in. + groups: inventory groups ``localhost`` joins. + applications: the host_vars ``applications`` block. + """ + write( + root / "host_vars/localhost.yml", + json.dumps({"applications": applications}), + ) + children = {group: {"hosts": {"localhost": {}}} for group in groups} + return write(root / "devices.yml", json.dumps({"all": {"children": children}})) + + +def declaration( + url: str, version: str | None = "v1.0.0", path: str = ROLE_PATH +) -> dict: + """The ``applications.`` block that declares a custom role. + + Args: + url: ``source.url``. + version: ``source.version``; ``None`` leaves it out. + path: ``source.path``. + """ + source = {"url": url, "path": path} + if version is not None: + source["version"] = version + return {"custom": True, "source": source} + + +def roles_dir(root: Path, *in_tree: str) -> Path: + """A core roles/ holding the given in-tree role directories. + + Args: + root: directory the roles/ is created in. + in_tree: names of in-tree roles to create. + """ + roles = root / "roles" + roles.mkdir(parents=True) + for name in in_tree: + (roles / name).mkdir() + return roles diff --git a/tests/unit/python/cli/administration/roles/clean/__init__.py b/tests/unit/python/cli/administration/roles/clean/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/cli/administration/roles/clean/test_main.py b/tests/unit/python/cli/administration/roles/clean/test_main.py new file mode 100644 index 0000000000..c3b921fc3c --- /dev/null +++ b/tests/unit/python/cli/administration/roles/clean/test_main.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import contextlib +import io +import unittest +from unittest import mock + +import cli.administration.roles.clean.__main__ as clean_main + + +class TestCleanMain(unittest.TestCase): + def test_reports_the_roles_it_removed_and_the_entries_it_left(self): + lines = [ + "web-app-cst-a: removed", + "web-app-cst-hand: left in place, the pull did not place it", + ] + with ( + mock.patch.object(clean_main, "clean", return_value=lines) as clean, + contextlib.redirect_stdout(io.StringIO()) as out, + ): + self.assertEqual(0, clean_main.main([])) + clean.assert_called_once_with(clean_main.ROLES_DIR, clean_main.WORK_DIR) + self.assertEqual( + "custom role web-app-cst-a: removed\n" + "custom role web-app-cst-hand: left in place, the pull did not place it\n", + out.getvalue(), + ) + + def test_a_filesystem_error_exits_with_it(self): + with ( + mock.patch.object(clean_main, "clean", side_effect=OSError("busy")), + self.assertRaises(SystemExit) as exit_, + ): + clean_main.main([]) + self.assertIn("busy", str(exit_.exception.code)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/cli/administration/roles/list/__init__.py b/tests/unit/python/cli/administration/roles/list/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/cli/administration/roles/list/test_main.py b/tests/unit/python/cli/administration/roles/list/test_main.py new file mode 100644 index 0000000000..7e89a24a5e --- /dev/null +++ b/tests/unit/python/cli/administration/roles/list/test_main.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import contextlib +import io +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import cli.administration.roles.list.__main__ as list_main +from cli.administration.roles.marker import MARKER_FILE, marker_text + +ORIGIN = "https://example.invalid/w.git ansible/roles/web-app-cst-a @ v1 (commit abc)" + + +class TestListMain(unittest.TestCase): + def test_prints_each_placed_role_with_source_and_commit(self): + with tempfile.TemporaryDirectory() as tmp: + roles = Path(tmp) + marker = roles / "web-app-cst-a" / MARKER_FILE + marker.parent.mkdir() + marker.write_text( + marker_text("web-app-cst-a", "web-app-cst-a", ORIGIN), encoding="utf-8" + ) + (roles / "web-app-cst-hand").mkdir() + (roles / "web-app-nextcloud").mkdir() + with ( + mock.patch.object(list_main, "ROLES_DIR", roles), + contextlib.redirect_stdout(io.StringIO()) as out, + ): + self.assertEqual(0, list_main.main([])) + self.assertEqual(f"web-app-cst-a: {ORIGIN}\n", out.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/cli/administration/roles/pull/__init__.py b/tests/unit/python/cli/administration/roles/pull/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/cli/administration/roles/pull/test_command.py b/tests/unit/python/cli/administration/roles/pull/test_command.py new file mode 100644 index 0000000000..6412c37d9e --- /dev/null +++ b/tests/unit/python/cli/administration/roles/pull/test_command.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import contextlib +import io +import tarfile +import unittest +from pathlib import Path +from unittest import mock + +from yaml.constructor import ConstructorError + +from cli.administration.roles.paths import WORK_DIR +from cli.administration.roles.pull import command +from utils.cache import ROLES_DIR +from utils.roles.custom import CustomRoleError + + +class TestPullCommand(unittest.TestCase): + @mock.patch.object( + command, "pull", return_value=["web-app-cst-demo: repo path @ v1 (commit abc)"] + ) + def test_pulls_the_inventory_into_the_core_roles(self, pull): + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(0, command.main(["inv/devices.yml"])) + pull.assert_called_once_with(Path("inv/devices.yml"), ROLES_DIR, WORK_DIR) + self.assertIn("web-app-cst-demo: repo path @ v1 (commit abc)", out.getvalue()) + + @mock.patch.object(command, "pull", return_value=[]) + def test_nothing_declared_or_placed_prints_nothing(self, _pull): + with contextlib.redirect_stdout(io.StringIO()) as out: + command.main(["inv/devices.yml"]) + self.assertEqual("", out.getvalue()) + + def test_unpacking_parsing_and_filesystem_errors_exit_1_naming_them(self): + for error in ( + tarfile.AbsoluteLinkError(tarfile.TarInfo(".infinito-custom-role")), + ConstructorError(None, None, "found an undefined tag in inv/hosts.yml"), + PermissionError(13, "Permission denied", "roles/web-app-cst-demo"), + ): + with self.subTest(error=type(error).__name__): + with ( + mock.patch.object(command, "pull", side_effect=error), + contextlib.redirect_stderr(io.StringIO()) as err, + self.assertRaises(SystemExit) as exit_, + ): + command.main(["inv/devices.yml"]) + self.assertEqual(1, exit_.exception.code) + self.assertIn(str(error), err.getvalue()) + + @mock.patch.object( + command, + "pull", + side_effect=CustomRoleError( + "web-app-cst-demo: git fetch of v1 failed: fatal: unable to access " + "'https://deploy:s3cr3t@example.invalid/w.git/'" + ), + ) + def test_credentials_in_a_url_are_never_echoed(self, _pull): + with ( + contextlib.redirect_stderr(io.StringIO()) as err, + self.assertRaises(SystemExit), + ): + command.main(["inv/devices.yml"]) + self.assertNotIn("s3cr3t", err.getvalue()) + self.assertIn("web-app-cst-demo: git fetch of v1 failed", err.getvalue()) + + @mock.patch.object( + command, "pull", side_effect=CustomRoleError("web-app-cst-demo: refused") + ) + def test_refused_pull_exits_1_with_the_reason(self, _pull): + with ( + contextlib.redirect_stderr(io.StringIO()) as err, + self.assertRaises(SystemExit) as exit_, + ): + command.main(["inv/devices.yml"]) + self.assertEqual(1, exit_.exception.code) + self.assertIn("web-app-cst-demo: refused", err.getvalue()) + + def test_inventory_argument_is_required(self): + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + command.main([]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/cli/administration/roles/test_declaration.py b/tests/unit/python/cli/administration/roles/test_declaration.py new file mode 100644 index 0000000000..8fc828bf76 --- /dev/null +++ b/tests/unit/python/cli/administration/roles/test_declaration.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from cli.administration.roles.declaration import Source, read_declarations + +from ._fixtures import ROLE, ROLE_PATH, declaration, inventory, roles_dir, write + +GIT_URL = "https://github.com/acme/widgets.git" + + +class TestReadDeclarations(unittest.TestCase): + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + self.roles = roles_dir(self.root, "web-app-nextcloud") + + def read(self, applications: dict, groups: tuple[str, ...] = (ROLE,)): + inv = inventory(self.root / "inventory", list(groups), applications) + return read_declarations(inv, self.roles) + + def assert_refused(self, applications: dict, *fragments: str, groups=(ROLE,)): + declared, errors = self.read(applications, groups) + message = "\n".join(errors) + for fragment in fragments: + self.assertIn(fragment, message) + return declared + + def test_complete_declaration_is_read(self): + declared, errors = self.read({ROLE: declaration(GIT_URL)}) + self.assertEqual([], errors) + self.assertEqual({ROLE: Source(GIT_URL, "v1.0.0", ROLE_PATH)}, declared) + + def test_ssh_url_is_accepted(self): + _declared, errors = self.read( + {ROLE: declaration("git@github.com:acme/widgets.git")} + ) + self.assertEqual([], errors) + + def test_ssh_config_host_alias_is_accepted(self): + _declared, errors = self.read( + {ROLE: declaration("github-store:acme/widgets.git")} + ) + self.assertEqual([], errors) + + def test_plain_http_url_is_refused(self): + self.assert_refused( + {ROLE: declaration("http://github.com/acme/widgets.git")}, + ROLE, + "source.url is neither", + ) + + def test_missing_url_names_group_and_field(self): + applications = { + ROLE: {"custom": True, "source": {"version": "v1", "path": ROLE_PATH}} + } + declared = self.assert_refused(applications, ROLE, "source.url is missing") + self.assertEqual({}, declared) + + def test_missing_version_of_a_git_url_names_the_field(self): + self.assert_refused( + {ROLE: declaration(GIT_URL, version=None)}, + ROLE, + "source.version is missing", + ) + + def test_missing_path_names_the_field(self): + applications = { + ROLE: {"custom": True, "source": {"url": GIT_URL, "version": "v1"}} + } + self.assert_refused(applications, ROLE, "source.path is missing") + + def test_missing_source_names_the_field(self): + self.assert_refused( + {ROLE: {"custom": True}}, ROLE, f"applications.{ROLE}.source is missing" + ) + + def test_local_directory_needs_no_version(self): + declared, errors = self.read({ROLE: declaration(str(self.root), version=None)}) + self.assertEqual([], errors) + self.assertTrue(declared[ROLE].local) + self.assertIsNone(declared[ROLE].version) + + def test_url_that_is_neither_git_nor_absolute_is_refused(self): + self.assert_refused( + {ROLE: declaration("checkouts/widgets")}, ROLE, "source.url is neither" + ) + + def test_url_with_credentials_is_refused_without_echoing_it(self): + for url in ( + "https://deploy:s3cr3t@github.com/acme/widgets.git", + "https://ghp_s3cr3t@github.com/acme/widgets.git", + "ssh://git:s3cr3t@github.com/acme/widgets.git", + ): + with self.subTest(url=url.split("@", 1)[1]): + declared, errors = self.read({ROLE: declaration(url)}) + message = "\n".join(errors) + self.assertIn( + f"{ROLE}: applications.{ROLE}.source.url embeds credentials", + message, + ) + self.assertNotIn("s3cr3t", message) + self.assertEqual({}, declared) + + def test_ssh_user_without_password_is_no_credential(self): + _declared, errors = self.read( + {ROLE: declaration("ssh://git@github.com/acme/widgets.git")} + ) + self.assertEqual([], errors) + + def test_jinja_in_a_source_field_is_refused(self): + applications = {ROLE: declaration("{{ lookup('env', 'ROLE_URL') }}")} + self.assert_refused(applications, ROLE, "source.url holds a Jinja expression") + + def test_jinja_in_custom_is_refused(self): + applications = {ROLE: {**declaration(GIT_URL), "custom": "{{ true }}"}} + declared = self.assert_refused( + applications, ROLE, f"{ROLE}.custom holds a Jinja expression" + ) + self.assertEqual({}, declared) + + def test_unknown_source_field_is_refused(self): + applications = {ROLE: declaration(GIT_URL)} + applications[ROLE]["source"]["tag"] = "v1" + self.assert_refused(applications, ROLE, "source.tag is not a source field") + + def test_path_leaving_the_repository_is_refused(self): + applications = {ROLE: declaration(GIT_URL, path=f"../{ROLE}")} + self.assert_refused(applications, ROLE, "relative path inside the repository") + + def test_directory_name_other_than_the_group_names_both(self): + applications = {ROLE: declaration(GIT_URL, path="ansible/roles/web-app-demo")} + self.assert_refused( + applications, "'web-app-demo'", f"'{ROLE}'", "never renames" + ) + + def test_group_without_custom_true_is_refused(self): + self.assert_refused( + {}, f"{ROLE}: inventory group without applications.{ROLE}.custom: true" + ) + + def test_custom_false_counts_as_undeclared(self): + applications = {ROLE: {**declaration(GIT_URL), "custom": False}} + declared = self.assert_refused(applications, f"{ROLE}: inventory group without") + self.assertEqual({}, declared) + + def test_custom_true_outside_the_prefix_is_refused(self): + applications = {"web-app-demo": declaration(GIT_URL)} + self.assert_refused( + applications, "web-app-demo: custom: true marks a group outside", groups=() + ) + + def test_entity_name_of_an_in_tree_role_is_refused(self): + applications = { + "web-app-cst-nextcloud": declaration( + GIT_URL, path="roles/web-app-cst-nextcloud" + ) + } + self.assert_refused( + applications, + "web-app-cst-nextcloud: entity name 'nextcloud'", + "in-tree role web-app-nextcloud", + groups=("web-app-cst-nextcloud",), + ) + + def test_declarations_that_differ_between_files_are_refused(self): + write( + self.root / "inventory/group_vars/all.yml", + json.dumps( + {"applications": {ROLE: declaration(GIT_URL, version="v2.0.0")}} + ), + ) + self.assert_refused( + {ROLE: declaration(GIT_URL)}, f"applications.{ROLE}.source differs between" + ) + + def read_yaml(self, host_vars: str): + base = self.root / "inventory" + write(base / "host_vars/localhost.yml", host_vars) + devices = {"all": {"children": {ROLE: {"hosts": {"localhost": {}}}}}} + return read_declarations( + write(base / "devices.yml", json.dumps(devices)), self.roles + ) + + def test_tagged_custom_is_refused_naming_the_tag(self): + declared, errors = self.read_yaml( + f"applications:\n {ROLE}:\n custom: !vault |\n" + " $ANSIBLE_VAULT;1.1;AES256\n 6162\n" + ) + self.assertEqual({}, declared) + self.assertIn( + f"{ROLE}: applications.{ROLE}.custom holds a !vault value", + "\n".join(errors), + ) + + def test_tagged_source_field_is_refused_naming_the_tag(self): + declared, errors = self.read_yaml( + f"applications:\n {ROLE}:\n custom: true\n source:\n" + f" url: {GIT_URL}\n version: !unsafe v1.0.0\n path: {ROLE_PATH}\n" + ) + self.assertEqual({}, declared) + self.assertIn( + f"{ROLE}: applications.{ROLE}.source.version holds a !unsafe value", + "\n".join(errors), + ) + + def test_unparsable_file_declares_nothing_and_is_the_only_error(self): + broken = write(self.root / "inventory/group_vars/all.yml", "applications: [\n") + declared, errors = self.read({ROLE: declaration(GIT_URL)}) + self.assertEqual({}, declared) + self.assertEqual(1, len(errors)) + self.assertTrue(errors[0].startswith(f"{broken}: cannot read"), errors[0]) + + def test_settings_without_custom_in_another_file_do_not_conflict(self): + write( + self.root / "inventory/group_vars/all.yml", + json.dumps( + {"applications": {ROLE: {"services": {"web": {"enabled": True}}}}} + ), + ) + _declared, errors = self.read({ROLE: declaration(GIT_URL)}) + self.assertEqual([], errors) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/cli/administration/roles/test_marker.py b/tests/unit/python/cli/administration/roles/test_marker.py new file mode 100644 index 0000000000..5cf799cd43 --- /dev/null +++ b/tests/unit/python/cli/administration/roles/test_marker.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +from cli.administration.roles.marker import ( + MARKER_FILE, + custom_entries, + is_placed, + marker_text, + placed_roles, + provenance, + write_marker, +) + +from ._fixtures import write + +ORIGIN = "https://example.invalid/w.git ansible/roles/web-app-cst-a @ v1 (commit abc)" +PLACED = "web-app-cst-a" + + +class TestMarker(unittest.TestCase): + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.roles = Path(tmp.name) / "roles" + write(self.roles / PLACED / MARKER_FILE, marker_text(PLACED, PLACED, ORIGIN)) + write(self.roles / "web-app-cst-hand" / "mine.yml", "mine: true\n") + write(self.roles / "web-app-cst-forged" / MARKER_FILE, "something else\n") + shutil.copytree(self.roles / PLACED, self.roles / "web-app-cst-copy") + (self.roles / "web-app-cst-linked" / "x").mkdir(parents=True) + (self.roles / "web-app-cst-linked" / MARKER_FILE).symlink_to( + self.roles / PLACED / MARKER_FILE + ) + write( + self.roles / "web-app-nextcloud" / MARKER_FILE, + marker_text("web-app-nextcloud", "web-app-nextcloud", ORIGIN), + ) + (self.roles / "web-app-cst-link").symlink_to(self.roles / PLACED) + + def test_only_a_directory_whose_marker_names_it_is_placed(self): + self.assertTrue(is_placed(self.roles / PLACED)) + self.assertFalse(is_placed(self.roles / "web-app-cst-hand")) + self.assertFalse(is_placed(self.roles / "web-app-cst-forged")) + + def test_a_copy_under_another_name_is_not_placed(self): + self.assertFalse(is_placed(self.roles / "web-app-cst-copy")) + + def test_a_marker_that_is_a_symbolic_link_is_not_read(self): + self.assertFalse(is_placed(self.roles / "web-app-cst-linked")) + + def test_a_marker_that_is_a_fifo_is_not_placed(self): + fifo = self.roles / "web-app-cst-fifo" + fifo.mkdir() + os.mkfifo(fifo / MARKER_FILE) + self.assertFalse(is_placed(fifo)) + + def test_a_symbolic_link_to_a_placed_role_is_not_placed(self): + self.assertFalse(is_placed(self.roles / "web-app-cst-link")) + + def test_provenance_reads_back_what_the_pull_recorded(self): + self.assertEqual(ORIGIN, provenance(self.roles / PLACED)) + + def test_custom_entries_are_every_prefixed_entry(self): + self.assertEqual( + [ + PLACED, + "web-app-cst-copy", + "web-app-cst-forged", + "web-app-cst-hand", + "web-app-cst-link", + "web-app-cst-linked", + ], + [entry.name for entry in custom_entries(self.roles)], + ) + + def test_placed_roles_are_the_custom_entries_whose_marker_names_them(self): + self.assertEqual([PLACED], [entry.name for entry in placed_roles(self.roles)]) + + +class TestWriteMarker(unittest.TestCase): + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + self.role = self.root / PLACED + self.role.mkdir() + + def test_marker_names_the_directory_and_the_group(self): + write_marker(self.role, PLACED, ORIGIN) + self.assertTrue(is_placed(self.role)) + self.assertEqual(ORIGIN, provenance(self.role)) + + def test_a_symbolic_link_under_the_marker_name_is_replaced_not_followed(self): + victim = write(self.root / "victim.yml", "victim: true\n") + (self.role / MARKER_FILE).symlink_to(victim) + write_marker(self.role, PLACED, ORIGIN) + text = victim.read_text(encoding="utf-8") # nocheck: cache-read tempdir + self.assertEqual("victim: true\n", text) + self.assertFalse((self.role / MARKER_FILE).is_symlink()) + self.assertTrue(is_placed(self.role)) + + def test_a_directory_under_the_marker_name_is_replaced(self): + write(self.role / MARKER_FILE / "nested.yml", "nested: true\n") + write_marker(self.role, PLACED, ORIGIN) + self.assertTrue((self.role / MARKER_FILE).is_file()) + self.assertTrue(is_placed(self.role)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/cli/administration/roles/test_placement.py b/tests/unit/python/cli/administration/roles/test_placement.py new file mode 100644 index 0000000000..03ca58c399 --- /dev/null +++ b/tests/unit/python/cli/administration/roles/test_placement.py @@ -0,0 +1,489 @@ +from __future__ import annotations + +import re +import shutil +import tempfile +import threading +import unittest +from pathlib import Path +from unittest import mock + +from cli.administration.roles import placement +from cli.administration.roles.marker import MARKER_FILE, is_placed +from cli.administration.roles.placement import clean, pull +from utils.cache.files import read_text +from utils.roles.custom import CustomRoleError +from utils.roles.mapping import ( + ROLE_FILE_META_MAIN, + ROLE_FILE_TASKS_MAIN, + ROLE_FILE_VARS_MAIN, +) + +from ._fixtures import ( + ROLE, + ROLE_PATH, + declaration, + git, + inventory, + local_role, + role_repo, + roles_dir, + write, +) + +OTHER = "web-app-cst-other" +COPY = "web-app-cst-mywork" +LEFT = "left in place, the pull did not place it" + + +class _PullFixture: + """Mixin with a core roles/, a role repository and a pull helper. + + Combined with ``unittest.TestCase`` on each concrete class so the + test-discovery lint sees the ``TestCase`` base directly. + """ + + def setUp(self) -> None: + super().setUp() + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + self.roles = roles_dir(self.root, "web-app-nextcloud") + self.work = self.root / "build" / "custom-roles" + self.repo = role_repo(self.root) + self.inventories = 0 + + def next_inventory(self) -> Path: + self.inventories += 1 + return self.root / f"inventory-{self.inventories}" + + def pull(self, applications: dict, groups: tuple[str, ...] = (ROLE,)) -> list[str]: + inv = inventory(self.next_inventory(), list(groups), applications) + return pull(inv, self.roles, self.work) + + def pull_yaml(self, devices: str, host_vars: str) -> list[str]: + base = self.next_inventory() + write(base / "host_vars/localhost.yml", host_vars) + return pull(write(base / "devices.yml", devices), self.roles, self.work) + + def placed(self) -> list[str]: + return sorted(path.name for path in self.roles.iterdir()) + + def leftovers(self) -> list[Path]: + if not self.work.exists(): + return [] + return [path for path in self.work.iterdir() if path.is_dir()] + + def tag(self, name: str) -> None: + git(self.repo, "add", "-A") + git(self.repo, "commit", "-q", "-m", name) + git(self.repo, "tag", name) + + +class TestPullPlacement(_PullFixture, unittest.TestCase): + def test_role_is_placed_at_its_tag_with_the_resolved_commit(self): + lines = self.pull({ROLE: declaration(str(self.repo))}) + self.assertEqual( + "release: v1\n", read_text(str(self.roles / ROLE / ROLE_FILE_TASKS_MAIN)) + ) + self.assertTrue(is_placed(self.roles / ROLE)) + self.assertRegex( + lines[0], rf"^{ROLE}: .* @ v1\.0\.0 \(commit [0-9a-f]{{40}}\)$" + ) + + def test_placed_role_carries_no_git_metadata(self): + self.pull({ROLE: declaration(str(self.repo), path=ROLE_PATH)}) + self.assertFalse((self.roles / ROLE / ".git").exists()) + + def test_local_working_tree_is_placed_without_a_version(self): + write(self.repo / ROLE_PATH / ROLE_FILE_TASKS_MAIN, "release: uncommitted\n") + self.pull({ROLE: declaration(str(self.repo), version=None)}) + self.assertEqual( + "release: uncommitted\n", + read_text(str(self.roles / ROLE / ROLE_FILE_TASKS_MAIN)), + ) + + def test_pull_again_replaces_the_placed_content(self): + self.pull({ROLE: declaration(str(self.repo))}) + write(self.roles / ROLE / "stale.yml", "stale: true\n") + self.pull({ROLE: declaration(str(self.repo))}) + self.assertFalse((self.roles / ROLE / "stale.yml").exists()) + self.assertTrue((self.roles / ROLE / ROLE_FILE_VARS_MAIN).is_file()) + + def test_role_the_inventory_no_longer_declares_is_removed(self): + self.pull({ROLE: declaration(str(self.repo))}) + lines = self.pull({}, groups=()) + self.assertEqual(["web-app-nextcloud"], self.placed()) + self.assertIn(f"{ROLE}: removed, the inventory no longer declares it", lines) + + def test_in_tree_roles_are_never_touched(self): + self.pull({ROLE: declaration(str(self.repo))}) + self.assertEqual([ROLE, "web-app-nextcloud"], self.placed()) + + def test_no_staging_or_leftover_directory_remains(self): + self.pull({ROLE: declaration(str(self.repo))}) + self.pull({ROLE: declaration(str(self.repo))}) + self.assertEqual([ROLE, "web-app-nextcloud"], self.placed()) + self.assertEqual([], self.leftovers()) + + def test_undeclared_directory_the_pull_did_not_place_is_left_alone(self): + write(self.roles / OTHER / "keep.yml", "keep: true\n") + lines = self.pull({ROLE: declaration(str(self.repo))}) + self.assertTrue((self.roles / OTHER / "keep.yml").is_file()) + self.assertIn(f"{OTHER}: {LEFT}", lines) + + +class TestPullRefusal(_PullFixture, unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.pull({ROLE: declaration(str(self.repo))}) + write(self.roles / ROLE / "sentinel.yml", "placed: true\n") + + def assert_untouched(self) -> None: + self.assertTrue((self.roles / ROLE / "sentinel.yml").is_file()) + self.assertEqual([ROLE, "web-app-nextcloud"], self.placed()) + self.assertEqual([], self.leftovers()) + + def test_branch_as_version_is_refused_before_anything_changes(self): + with self.assertRaisesRegex(CustomRoleError, "'feature' names a branch"): + self.pull({ROLE: declaration(str(self.repo), version="feature")}) + self.assert_untouched() + + def test_application_id_other_than_the_group_names_both(self): + other = role_repo(self.root / "other", application_id="web-app-demo") + with self.assertRaisesRegex( + CustomRoleError, + f"application_id is 'web-app-demo', but the group is '{ROLE}'", + ): + self.pull({ROLE: declaration(str(other))}) + self.assert_untouched() + + def test_missing_application_id_is_refused(self): + write(self.repo / ROLE_PATH / ROLE_FILE_VARS_MAIN, "other: value\n") + with self.assertRaisesRegex(CustomRoleError, "application_id is None"): + self.pull({ROLE: declaration(str(self.repo), version=None)}) + self.assert_untouched() + + def test_source_without_meta_main_is_refused(self): + (self.repo / ROLE_PATH / ROLE_FILE_META_MAIN).unlink() + with self.assertRaisesRegex(CustomRoleError, "no role directory"): + self.pull({ROLE: declaration(str(self.repo), version=None)}) + self.assert_untouched() + + def test_invalid_declaration_is_refused_before_anything_changes(self): + with self.assertRaisesRegex(CustomRoleError, "source.path is missing"): + self.pull( + { + ROLE: { + "custom": True, + "source": {"url": str(self.repo), "version": "v1.0.0"}, + } + } + ) + self.assert_untouched() + + def test_path_that_is_no_directory_in_the_source_is_refused(self): + with self.assertRaisesRegex(CustomRoleError, "source.path is not a directory"): + self.pull({ROLE: declaration(str(self.repo), path=f"absent/{ROLE}")}) + self.assert_untouched() + + def test_local_source_inside_its_placement_is_refused(self): + with self.assertRaisesRegex(CustomRoleError, "never deletes its own source"): + self.pull( + {ROLE: declaration(str(self.root), version=None, path=f"roles/{ROLE}")} + ) + self.assert_untouched() + + def test_a_failing_role_leaves_every_placed_role_unchanged(self): + broken = { + "custom": True, + "source": {"url": str(self.root / "absent"), "path": f"x/{OTHER}"}, + } + with self.assertRaisesRegex(CustomRoleError, OTHER): + self.pull( + {ROLE: declaration(str(self.repo)), OTHER: broken}, groups=(ROLE, OTHER) + ) + self.assert_untouched() + + def test_missing_inventory_is_refused(self): + with self.assertRaisesRegex(CustomRoleError, "neither a file nor a directory"): + pull(self.root / "absent/devices.yml", self.roles, self.work) + self.assert_untouched() + + def test_unparsable_vars_file_stops_the_pull_naming_it(self): + inv = inventory(self.next_inventory(), [], {}) + broken = write(inv.parent / "group_vars/all.yml", "applications:\n x: [open\n") + with self.assertRaisesRegex( + CustomRoleError, rf"{re.escape(str(broken))}: cannot read" + ): + pull(inv, self.roles, self.work) + self.assert_untouched() + + def test_vars_file_that_is_no_utf8_stops_the_pull_naming_it(self): + inv = inventory(self.next_inventory(), [], {}) + latin = inv.parent / "group_vars/all.yml" + latin.parent.mkdir() + latin.write_bytes("applications: {caf\xe9: {}}\n".encode("latin-1")) + with self.assertRaisesRegex( + CustomRoleError, rf"{re.escape(str(latin))}: cannot read" + ): + pull(inv, self.roles, self.work) + self.assert_untouched() + + def test_inventory_file_that_is_no_utf8_stops_the_pull_naming_it(self): + inv = inventory(self.next_inventory(), [], {}) + inv.write_bytes("all:\n hosts:\n caf\xe9: {}\n".encode("latin-1")) + with self.assertRaisesRegex( + CustomRoleError, rf"{re.escape(str(inv))}: cannot read" + ): + pull(inv, self.roles, self.work) + self.assert_untouched() + + +class TestPullInventoryForms(_PullFixture, unittest.TestCase): + def test_directory_inventory_is_read_like_a_file_inventory(self): + inv = inventory( + self.next_inventory(), [ROLE], {ROLE: declaration(str(self.repo))} + ) + pull(inv.parent, self.roles, self.work) + self.assertTrue(is_placed(self.roles / ROLE)) + + def test_group_of_a_hosts_file_in_a_directory_inventory_needs_custom_true(self): + base = self.next_inventory() + write(base / "hosts", f"[{ROLE}]\nlocalhost\n") + write(base / "README.md", "[web-app-cst-ignored]\n") + write(base / "host_vars/localhost.yml", "applications: {}\n") + with self.assertRaises(CustomRoleError) as refused: + pull(base, self.roles, self.work) + self.assertIn(f"{ROLE}: inventory group without", str(refused.exception)) + self.assertNotIn("web-app-cst-ignored", str(refused.exception)) + + def test_host_list_declares_nothing_and_changes_nothing(self): + self.pull({ROLE: declaration(str(self.repo))}) + self.assertEqual([], pull(Path("localhost,"), self.roles, self.work)) + self.assertTrue(is_placed(self.roles / ROLE)) + + +class TestPullAnsibleTags(_PullFixture, unittest.TestCase): + DEVICES = ( + "all:\n" + " hosts:\n" + " localhost:\n" + " motd: !unsafe '{{ literal }}'\n" + " children:\n" + f" {ROLE}:\n" + " hosts:\n" + " localhost: {}\n" + ) + + def host_vars(self, url: str) -> str: + return ( + "secret: !vault |\n" + " $ANSIBLE_VAULT;1.1;AES256\n" + " 6162\n" + "applications:\n" + f" {ROLE}:\n" + " custom: true\n" + " source:\n" + f" url: {url}\n" + " version: v1.0.0\n" + f" path: {ROLE_PATH}\n" + ) + + def test_tagged_values_elsewhere_in_the_inventory_keep_the_pull_working(self): + self.pull_yaml(self.DEVICES, self.host_vars(str(self.repo))) + self.assertTrue(is_placed(self.roles / ROLE)) + + def test_tagged_source_value_is_refused_like_a_jinja_value(self): + with self.assertRaisesRegex( + CustomRoleError, + f"{ROLE}: applications.{ROLE}.source.url holds a !unsafe value", + ): + self.pull_yaml(self.DEVICES, self.host_vars(f"!unsafe '{self.repo}'")) + self.assertEqual(["web-app-nextcloud"], self.placed()) + + +class TestPullOwnership(_PullFixture, unittest.TestCase): + def test_symbolic_link_under_a_declared_name_is_refused_and_kept(self): + target = self.root / "elsewhere" + write(target / "keep.yml", "keep: true\n") + (self.roles / ROLE).symlink_to(target) + with self.assertRaisesRegex( + CustomRoleError, f"{ROLE}: .* is a symbolic link .* remove the link" + ): + self.pull({ROLE: declaration(str(self.repo))}) + self.assertTrue((self.roles / ROLE).is_symlink()) + self.assertTrue((target / "keep.yml").is_file()) + + def test_hand_placed_directory_under_a_declared_name_is_refused_and_kept(self): + write(self.roles / ROLE / "mine.yml", "mine: true\n") + with self.assertRaisesRegex( + CustomRoleError, f"{ROLE}: .* was not placed by a pull" + ): + self.pull({ROLE: declaration(str(self.repo))}) + self.assertTrue((self.roles / ROLE / "mine.yml").is_file()) + self.assertFalse((self.roles / ROLE / MARKER_FILE).exists()) + + def test_copy_of_a_placed_role_is_left_in_place_and_reported(self): + self.pull({ROLE: declaration(str(self.repo))}) + shutil.copytree(self.roles / ROLE, self.roles / COPY) + lines = self.pull({}, groups=()) + self.assertEqual([COPY, "web-app-nextcloud"], self.placed()) + self.assertIn(f"{COPY}: {LEFT}", lines) + + def test_declared_name_that_meets_a_copy_is_refused_and_kept(self): + self.pull({ROLE: declaration(str(self.repo))}) + shutil.copytree(self.roles / ROLE, self.roles / COPY) + write(self.roles / COPY / "mine.yml", "mine: true\n") + checkout = local_role(self.root, COPY) + mine = declaration(str(checkout), version=None, path=f"ansible/roles/{COPY}") + with self.assertRaisesRegex( + CustomRoleError, f"{COPY}: .* was not placed by a pull" + ): + self.pull({ROLE: declaration(str(self.repo)), COPY: mine}, (ROLE, COPY)) + self.assertTrue((self.roles / COPY / "mine.yml").is_file()) + + +class TestPullMarker(_PullFixture, unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.in_tree = write( + self.roles / "web-app-nextcloud" / ROLE_FILE_TASKS_MAIN, "in-tree: true\n" + ) + + def assert_in_tree_untouched(self) -> None: + text = self.in_tree.read_text(encoding="utf-8") # nocheck: cache-read tempdir + self.assertEqual("in-tree: true\n", text) + self.assertEqual(["web-app-nextcloud"], self.placed()) + self.assertEqual([], self.leftovers()) + + def test_marker_link_in_a_local_source_is_refused_and_never_followed(self): + checkout = local_role(self.root, ROLE) + (checkout / "ansible/roles" / ROLE / MARKER_FILE).symlink_to(self.in_tree) + with self.assertRaisesRegex( + CustomRoleError, f"{ROLE}: .*{MARKER_FILE} is a symbolic link" + ): + self.pull( + { + ROLE: declaration( + str(checkout), version=None, path=f"ansible/roles/{ROLE}" + ) + } + ) + self.assert_in_tree_untouched() + + def test_relative_marker_link_in_a_tag_is_refused(self): + (self.repo / ROLE_PATH / MARKER_FILE).symlink_to(ROLE_FILE_TASKS_MAIN) + self.tag("v2.0.0") + with self.assertRaisesRegex( + CustomRoleError, f"{ROLE}: .*{MARKER_FILE} is a symbolic link" + ): + self.pull({ROLE: declaration(str(self.repo), version="v2.0.0")}) + self.assert_in_tree_untouched() + + def test_absolute_link_in_a_tag_is_refused_naming_the_group(self): + (self.repo / ROLE_PATH / MARKER_FILE).symlink_to(self.in_tree) + self.tag("v2.0.0") + with self.assertRaisesRegex(CustomRoleError, f"^{ROLE}: .*absolute"): + self.pull({ROLE: declaration(str(self.repo), version="v2.0.0")}) + self.assert_in_tree_untouched() + + +class TestPullCommit(_PullFixture, unittest.TestCase): + def test_interrupt_while_deleting_the_old_copy_keeps_the_new_one(self): + self.pull({ROLE: declaration(str(self.repo))}) + write(self.roles / ROLE / "stale.yml", "stale: true\n") + rmtree = shutil.rmtree + + def interrupted(path, *args, **kwargs): + if ".retired." in str(path): + raise KeyboardInterrupt + return rmtree(path, *args, **kwargs) + + with ( + mock.patch.object(placement.shutil, "rmtree", interrupted), + self.assertRaises(KeyboardInterrupt), + ): + self.pull({ROLE: declaration(str(self.repo))}) + self.assertTrue(is_placed(self.roles / ROLE)) + self.assertFalse((self.roles / ROLE / "stale.yml").exists()) + self.pull({ROLE: declaration(str(self.repo))}) + self.assertEqual([], self.leftovers()) + + def test_directory_that_appears_at_a_declared_name_during_the_pull_is_kept(self): + stage = placement.stage + + def stage_beside_a_hand_placed_copy(group, source, target): + write(self.roles / group / "mine.yml", "mine: true\n") + return stage(group, source, target) + + with ( + mock.patch.object(placement, "stage", stage_beside_a_hand_placed_copy), + self.assertRaisesRegex( + CustomRoleError, f"{ROLE}: .* was not placed by a pull" + ), + ): + self.pull({ROLE: declaration(str(self.repo))}) + self.assertTrue((self.roles / ROLE / "mine.yml").is_file()) + self.assertFalse(is_placed(self.roles / ROLE)) + self.assertEqual([], self.leftovers()) + + def test_clean_waits_until_a_running_pull_has_finished(self): + events: list[tuple[str, object]] = [] + cleaner = threading.Thread( + target=lambda: events.append(("cleaned", clean(self.roles, self.work))) + ) + stage = placement.stage + + def stage_while_a_clean_starts(group, source, target): + cleaner.start() + cleaner.join(timeout=0.5) + events.append(("clean still waiting", cleaner.is_alive())) + return stage(group, source, target) + + with mock.patch.object(placement, "stage", stage_while_a_clean_starts): + self.pull({ROLE: declaration(str(self.repo))}) + cleaner.join(timeout=30) + self.assertEqual(("clean still waiting", True), events[0]) + self.assertEqual("cleaned", events[1][0]) + self.assertEqual(["web-app-nextcloud"], self.placed()) + + +class TestClean(_PullFixture, unittest.TestCase): + def test_removes_only_the_roles_the_pull_placed_and_reports_the_rest(self): + checkout = local_role(self.root, OTHER) + self.pull( + { + ROLE: declaration(str(self.repo)), + OTHER: declaration( + str(checkout), version=None, path=f"ansible/roles/{OTHER}" + ), + }, + groups=(ROLE, OTHER), + ) + shutil.copytree(self.roles / ROLE, self.roles / COPY) + write(self.roles / "web-app-cst-hand" / "mine.yml", "mine: true\n") + target = self.root / "elsewhere" + write(target / "keep.yml", "keep: true\n") + (self.roles / "web-app-cst-link").symlink_to(target) + lines = clean(self.roles, self.work) + self.assertEqual( + [ + f"{ROLE}: removed", + f"web-app-cst-hand: {LEFT}", + f"web-app-cst-link: {LEFT}", + f"{COPY}: {LEFT}", + f"{OTHER}: removed", + ], + lines, + ) + self.assertEqual( + ["web-app-cst-hand", "web-app-cst-link", COPY, "web-app-nextcloud"], + self.placed(), + ) + self.assertTrue((target / "keep.yml").is_file()) + self.assertEqual([], self.leftovers()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/cli/administration/roles/test_source.py b/tests/unit/python/cli/administration/roles/test_source.py new file mode 100644 index 0000000000..e37dbe2f1b --- /dev/null +++ b/tests/unit/python/cli/administration/roles/test_source.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from cli.administration.roles import source as source_module +from cli.administration.roles.declaration import Source +from cli.administration.roles.source import check, stage +from utils.cache.files import read_text +from utils.roles.custom import CustomRoleError +from utils.roles.mapping import ROLE_FILE_TASKS_MAIN + +from ._fixtures import ROLE, ROLE_PATH, git, role_repo, roles_dir, write + + +class _SourceFixture: + """Mixin with a core roles/ and a role repository. + + Combined with ``unittest.TestCase`` on each concrete class so the + test-discovery lint sees the ``TestCase`` base directly. + """ + + def setUp(self) -> None: + super().setUp() + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + self.roles = roles_dir(self.root) + self.repo = role_repo(self.root) + + def check(self, source: Source, placements: tuple[Path, ...] = ()) -> None: + check(ROLE, source, list(placements), self.roles) + + +class TestCheckVersion(_SourceFixture, unittest.TestCase): + def test_tag_is_accepted(self): + self.check(Source(str(self.repo), "v1.0.0", ROLE_PATH)) + + def test_full_commit_sha_is_accepted(self): + sha = git(self.repo, "rev-parse", "v1.0.0").strip() + self.check(Source(str(self.repo), sha, ROLE_PATH)) + + def test_full_commit_sha_needs_no_ref_lookup(self): + self.check(Source("https://example.invalid/widgets.git", "a" * 40, ROLE_PATH)) + + def test_branch_is_refused(self): + with self.assertRaisesRegex( + CustomRoleError, f"{ROLE}: source.version 'feature' names a branch" + ): + self.check(Source(str(self.repo), "feature", ROLE_PATH)) + + def test_abbreviated_sha_is_refused(self): + sha = git(self.repo, "rev-parse", "--short", "v1.0.0").strip() + with self.assertRaisesRegex( + CustomRoleError, "neither a tag nor a full commit SHA" + ): + self.check(Source(str(self.repo), sha, ROLE_PATH)) + + def test_unknown_tag_is_refused(self): + with self.assertRaisesRegex(CustomRoleError, "'v9.9.9' is neither a tag"): + self.check(Source(str(self.repo), "v9.9.9", ROLE_PATH)) + + +class TestCheckLocalSource(_SourceFixture, unittest.TestCase): + def test_missing_directory_is_refused(self): + with self.assertRaisesRegex(CustomRoleError, "is not a directory"): + self.check(Source(str(self.root / "absent"), None, ROLE_PATH)) + + def test_own_placement_directory_is_refused_naming_the_path(self): + (self.roles / ROLE).mkdir() + placement = re.escape(str((self.roles / ROLE).resolve())) + with self.assertRaisesRegex( + CustomRoleError, f"{placement} lies in the placement directory" + ): + self.check(Source(str(self.root), None, f"roles/{ROLE}")) + + def test_another_placement_is_refused(self): + other = self.roles / "web-app-cst-other" + other.mkdir() + with self.assertRaisesRegex(CustomRoleError, "never deletes its own source"): + self.check( + Source(str(self.root), None, f"roles/{other.name}/{ROLE}"), (other,) + ) + + def test_placement_reached_through_a_symlink_is_refused(self): + (self.roles / ROLE).symlink_to(self.repo / ROLE_PATH) + with self.assertRaisesRegex(CustomRoleError, "never deletes its own source"): + self.check(Source(str(self.repo), None, ROLE_PATH)) + + def test_source_through_a_symlinked_checkout_is_refused(self): + alias = self.root / "alias" + alias.symlink_to(self.root) + with self.assertRaisesRegex(CustomRoleError, "never deletes its own source"): + self.check(Source(str(alias), None, f"roles/{ROLE}")) + + def test_checkout_outside_roles_is_accepted(self): + self.check(Source(str(self.repo), None, ROLE_PATH)) + + +class TestStage(_SourceFixture, unittest.TestCase): + def test_tag_stages_only_the_role_path_at_that_tag(self): + target = self.root / "staged" + origin = stage(ROLE, Source(str(self.repo), "v1.0.0", ROLE_PATH), target) + sha = git(self.repo, "rev-parse", "v1.0.0^{commit}").strip() + self.assertEqual("release: v1\n", read_text(str(target / ROLE_FILE_TASKS_MAIN))) + self.assertFalse((target / "README.md").exists()) + self.assertIn(f"(commit {sha})", origin) + + def test_full_commit_sha_is_staged(self): + sha = git(self.repo, "rev-parse", "v1.0.0^{commit}").strip() + target = self.root / "staged" + stage(ROLE, Source(str(self.repo), sha, ROLE_PATH), target) + self.assertEqual("release: v1\n", read_text(str(target / ROLE_FILE_TASKS_MAIN))) + + def test_local_directory_without_version_is_its_working_tree(self): + write(self.repo / ROLE_PATH / ROLE_FILE_TASKS_MAIN, "release: uncommitted\n") + (self.repo / ROLE_PATH / "__pycache__").mkdir() + target = self.root / "staged" + origin = stage(ROLE, Source(str(self.repo), None, ROLE_PATH), target) + self.assertEqual( + "release: uncommitted\n", read_text(str(target / ROLE_FILE_TASKS_MAIN)) + ) + self.assertFalse((target / "__pycache__").exists()) + self.assertIn("(working tree)", origin) + + def test_path_that_is_a_file_is_refused(self): + with self.assertRaisesRegex(CustomRoleError, "source.path is not a directory"): + stage(ROLE, Source(str(self.repo), "v1.0.0", "README.md"), self.root / "s") + + def test_missing_path_is_refused(self): + with self.assertRaisesRegex(CustomRoleError, "source.path is not a directory"): + stage( + ROLE, + Source(str(self.repo), "v1.0.0", f"absent/{ROLE}"), + self.root / "s", + ) + + def test_failed_fetch_names_the_group(self): + with self.assertRaisesRegex( + CustomRoleError, f"{ROLE}: git fetch of v1.0.0 failed" + ): + stage( + ROLE, + Source(str(self.root / "absent"), "v1.0.0", ROLE_PATH), + self.root / "staged", + ) + + +class TestGitNeverPrompts(unittest.TestCase): + def test_git_runs_detached_without_a_prompt_or_askpass(self): + failed = subprocess.CompletedProcess( + ["git"], 128, stdout=b"", stderr=b"fatal: could not read Username" + ) + source = Source("git@example.invalid:acme/widgets.git", "v1.0.0", ROLE_PATH) + with ( + tempfile.TemporaryDirectory() as tmp, + mock.patch.dict( + "os.environ", + { + "GIT_ASKPASS": "/bin/askpass", + "SSH_ASKPASS": "/bin/askpass", + "SSH_AUTH_SOCK": "/run/agent.sock", + }, + ), + mock.patch.object( + source_module.subprocess, "run", return_value=failed + ) as run, + self.assertRaisesRegex(CustomRoleError, "could not read Username"), + ): + stage(ROLE, source, Path(tmp) / "staged") + kwargs = run.call_args.kwargs + self.assertEqual("0", kwargs["env"]["GIT_TERMINAL_PROMPT"]) + self.assertEqual("never", kwargs["env"]["SSH_ASKPASS_REQUIRE"]) + self.assertNotIn("GIT_ASKPASS", kwargs["env"]) + self.assertNotIn("SSH_ASKPASS", kwargs["env"]) + self.assertEqual("/run/agent.sock", kwargs["env"]["SSH_AUTH_SOCK"]) + self.assertTrue(kwargs["start_new_session"]) + self.assertEqual(subprocess.DEVNULL, kwargs["stdin"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/tests/__init__.py b/tests/unit/python/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/tests/lint/__init__.py b/tests/unit/python/tests/lint/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/tests/lint/ansible/__init__.py b/tests/unit/python/tests/lint/ansible/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/tests/lint/ansible/roles/__init__.py b/tests/unit/python/tests/lint/ansible/roles/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/tests/lint/ansible/roles/test_readme.py b/tests/unit/python/tests/lint/ansible/roles/test_readme.py new file mode 100644 index 0000000000..ae3cb6506a --- /dev/null +++ b/tests/unit/python/tests/lint/ansible/roles/test_readme.py @@ -0,0 +1,69 @@ +"""The role README lint exempts a custom role from the Infinito.Nexus licence only.""" + +from __future__ import annotations + +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from tests.lint.ansible.roles import test_readme as lint +from utils.roles.mapping import ROLE_FILE_META_MAIN, ROLE_FILE_README + +AUTHOR = "Jane Doe" +IMPLEMENTED = f"Implemented by **{AUTHOR}**.\n" +PROJECT = ( + "Part of the [Infinito.Nexus Project](https://s.infinito.nexus/code) " + "and maintained by [Kevin Veen-Birkenbach](https://www.veen.world).\n" +) +INFINITO_LICENSE = ( + "Licensed under the [Infinito.Nexus Community License (Non-Commercial)]" + "(https://s.infinito.nexus/license).\n" +) +OWN_LICENSE = "Licensed under the [MIT License](https://opensource.org/licenses/MIT).\n" +TAIL_PROBLEM = "Credits tail does not match the canonical wording" + + +class TestCreditsLicence(unittest.TestCase): + def setUp(self) -> None: + tmp = TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.roles = Path(tmp.name) + + def _problems(self, role: str, *lines: str) -> list[str]: + readme = self.roles / role / ROLE_FILE_README + meta = readme.parent / ROLE_FILE_META_MAIN + meta.parent.mkdir(parents=True) + meta.write_text(f"galaxy_info:\n author: {AUTHOR}\n", encoding="utf-8") + text = "## Credits\n\n" + "".join(lines) + readme.write_text(text, encoding="utf-8") + return lint._validate_credits(text, readme) + + def test_a_custom_role_with_its_own_licence_passes(self) -> None: + self.assertEqual( + [], + self._problems("web-app-cst-demo", IMPLEMENTED, PROJECT, OWN_LICENSE), + ) + + def test_an_in_tree_role_without_the_infinito_licence_fails(self) -> None: + problems = self._problems("web-app-demo", IMPLEMENTED, PROJECT, OWN_LICENSE) + self.assertTrue(any(TAIL_PROBLEM in problem for problem in problems)) + + def test_an_in_tree_role_with_the_infinito_licence_passes(self) -> None: + self.assertEqual( + [], + self._problems("web-app-demo", IMPLEMENTED, PROJECT, INFINITO_LICENSE), + ) + + def test_a_custom_role_without_the_project_line_still_fails(self) -> None: + problems = self._problems("web-app-cst-demo", IMPLEMENTED, OWN_LICENSE) + self.assertTrue(any(TAIL_PROBLEM in problem for problem in problems)) + + def test_a_custom_role_still_needs_the_author_spot(self) -> None: + problems = self._problems( + "web-app-cst-demo", "Implemented by **Someone Else**.\n", PROJECT + ) + self.assertTrue(any("galaxy_info.author" in problem for problem in problems)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/tests/test_untracked_files_in_scope.py b/tests/unit/python/tests/test_untracked_files_in_scope.py new file mode 100644 index 0000000000..ef85777c49 --- /dev/null +++ b/tests/unit/python/tests/test_untracked_files_in_scope.py @@ -0,0 +1,83 @@ +"""Repository scans see a file before it is added, whichever runner lists the files.""" + +from __future__ import annotations + +import subprocess +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest import mock + +from tests.integration.infrastructure.compose import test_no_raw_docker as raw_docker +from tests.lint.filesystem.python import test_python_shebangs as shebangs +from tests.lint.repository import test_comments_valid as comments +from tests.lint.repository.documentation import test_todo_transparency as todo +from utils.cache import files as files_module + +TRACKED = "tracked.py" +UNTRACKED = "new.py" +FAILED = subprocess.CalledProcessError(128, "git") + + +class _ListingFixture: + """Mixin with a project root holding one tracked and one untracked file. + + Combined with ``unittest.TestCase`` on each concrete class so the + test-discovery lint sees the ``TestCase`` base directly. + """ + + def setUp(self) -> None: + super().setUp() + files_module._reset() + self.addCleanup(files_module._reset) + tmp = TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + for name in (TRACKED, UNTRACKED): + (self.root / name).write_text("", encoding="utf-8") + for module in (files_module, comments): + patcher = mock.patch.object(module, "PROJECT_ROOT", self.root) + patcher.start() + self.addCleanup(patcher.stop) + + def git(self, tracked: bytes | Exception, untracked: bytes | Exception): + def run(args, **_kwargs): + listing = untracked if "--others" in args else tracked + if isinstance(listing, Exception): + raise listing + return SimpleNamespace(stdout=listing) + + return mock.patch.object(files_module.subprocess, "run", side_effect=run) + + +class TestScansListUntrackedFiles(_ListingFixture, unittest.TestCase): + def test_every_scan_lists_a_file_that_is_not_added_yet(self): + scans = { + "test_python_shebangs": lambda: shebangs.TestPythonShebangs._tracked_files( + SimpleNamespace(repo_root=self.root) + ), + "test_todo_transparency": lambda: todo.tracked_files(self.root), + "test_no_raw_docker": lambda: raw_docker.git_ls_files(self.root), + } + for name, scan in scans.items(): + with ( + self.subTest(scan=name), + self.git(f"{TRACKED}\0".encode(), f"{UNTRACKED}\0".encode()), + ): + files_module._reset() + self.assertIn(self.root / UNTRACKED, scan()) + + +class TestCommentScanKeepsEachListing(_ListingFixture, unittest.TestCase): + def test_failing_tracked_listing_keeps_the_untracked_files(self): + with self.git(FAILED, f"{UNTRACKED}\0".encode()): + self.assertEqual([self.root / UNTRACKED], comments._all_targets()) + + def test_failing_untracked_listing_keeps_the_tracked_files(self): + with self.git(f"{TRACKED}\0".encode(), FAILED): + self.assertEqual([self.root / TRACKED], comments._all_targets()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/cache/test_files.py b/tests/unit/python/utils/cache/test_files.py index 466e4f9ef0..76b2eab7e8 100644 --- a/tests/unit/python/utils/cache/test_files.py +++ b/tests/unit/python/utils/cache/test_files.py @@ -8,9 +8,11 @@ the module so the real repo tree is never touched. from __future__ import annotations +import subprocess import unittest from pathlib import Path from tempfile import TemporaryDirectory +from types import SimpleNamespace from unittest.mock import patch from utils.cache import files as files_module @@ -20,7 +22,7 @@ from utils.cache.files import ( iter_project_files_with_content, read_text, ) -from utils.roles.mapping import ROLE_FILE_META_SERVICES +from utils.roles.mapping import ROLE_FILE_META_SERVICES, ROLE_FILE_TASKS_MAIN def _touch(path: Path, content: str = "") -> Path: @@ -177,6 +179,56 @@ class TestReset(_ProjectRootFixture, unittest.TestCase): self.assertEqual(read_text(str(self.root / "x.txt")), "v1-changed") +class TestRepositoryFiles(_ProjectRootFixture, unittest.TestCase): + TRACKED = f"roles/web-app-nextcloud/{ROLE_FILE_TASKS_MAIN}" + CUSTOM = f"roles/web-app-cst-demo/{ROLE_FILE_TASKS_MAIN}" + + def _git(self, *listings: bytes): + return patch.object( + files_module.subprocess, + "run", + side_effect=[SimpleNamespace(stdout=listing) for listing in listings], + ) + + def test_lists_placed_custom_roles_next_to_the_tracked_files(self) -> None: + _touch(self.root / self.TRACKED, "- y") + _touch(self.root / self.CUSTOM, "- x") + _touch(self.root / "roles/web-app-cst-demo/__pycache__/x.pyc", "") + with self._git(f"{self.TRACKED}\0".encode()): + self.assertEqual( + (self.CUSTOM, self.TRACKED), + files_module.repository_files(), + ) + + def test_lists_only_the_tracked_files_without_placed_custom_roles(self) -> None: + _touch(self.root / self.TRACKED, "- y") + _touch(self.root / "build/install.stamp", "") + with self._git(f"{self.TRACKED}\0".encode()): + self.assertEqual((self.TRACKED,), files_module.repository_files()) + + def test_untracked_adds_the_files_gitignore_does_not_exclude(self) -> None: + with self._git(f"{self.TRACKED}\0".encode(), b"new.py\0") as run: + self.assertEqual( + ("new.py", self.TRACKED), + files_module.repository_files(untracked=True), + ) + self.assertEqual( + ["--others", "--exclude-standard"], + run.call_args_list[1].args[0][-2:], + ) + + def test_a_git_failure_propagates(self) -> None: + with ( + patch.object( + files_module.subprocess, + "run", + side_effect=subprocess.CalledProcessError(128, "git"), + ), + self.assertRaises(subprocess.CalledProcessError), + ): + files_module.repository_files() + + class TestWalkIsKeyedOnTheRoot(_ProjectRootFixture, unittest.TestCase): def test_a_second_root_is_walked_rather_than_answered_from_the_first( self, diff --git a/tests/unit/python/utils/cache/test_gitignore.py b/tests/unit/python/utils/cache/test_gitignore.py new file mode 100644 index 0000000000..549da606d0 --- /dev/null +++ b/tests/unit/python/utils/cache/test_gitignore.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from utils.cache import gitignore +from utils.roles.custom import ROLES_GLOB +from utils.roles.mapping import ROLE_FILE_TASKS_MAIN + + +class TestLoadGitignorePatterns(unittest.TestCase): + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.addCleanup(gitignore._reset) + self.root = Path(tmp.name) + (self.root / ".gitignore").write_text( + f"# comment\n*.log\n{ROLES_GLOB}\n!keep.log\nbuild/*\n", encoding="utf-8" + ) + + def test_reads_plain_patterns(self): + patterns = gitignore.load_gitignore_patterns(str(self.root)) + self.assertEqual(("*.log", "build/*"), patterns) + + def test_pulled_custom_roles_stay_in_the_scanned_tree(self): + patterns = gitignore.load_gitignore_patterns(str(self.root)) + self.assertFalse( + gitignore.is_path_gitignored( + f"roles/web-app-cst-demo/{ROLE_FILE_TASKS_MAIN}", patterns + ) + ) + + def test_ignored_files_inside_a_custom_role_stay_ignored(self): + patterns = gitignore.load_gitignore_patterns(str(self.root)) + self.assertTrue( + gitignore.is_path_gitignored("roles/web-app-cst-demo/debug.log", patterns) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/cache/test_yaml.py b/tests/unit/python/utils/cache/test_yaml.py index cd948e5d93..240f4dd245 100644 --- a/tests/unit/python/utils/cache/test_yaml.py +++ b/tests/unit/python/utils/cache/test_yaml.py @@ -7,11 +7,13 @@ from pathlib import Path from tempfile import TemporaryDirectory from utils.cache.yaml import ( + TaggedValue, _reset_cache_for_tests, dump_yaml, invalidate, load_yaml, load_yaml_any, + load_yaml_tagged, ) @@ -158,5 +160,38 @@ class TestInvalidate(unittest.TestCase): invalidate(self.tmp / "never-loaded.yml") +class TestLoadYamlTagged(unittest.TestCase): + def setUp(self) -> None: + _reset_cache_for_tests() + self._tmp = TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.path = Path(self._tmp.name) / "inventory.yml" + self.path.write_text( + "motd: !unsafe '{{ literal }}'\n" + "secret: !vault |\n" + " $ANSIBLE_VAULT;1.1;AES256\n" + " 6162\n" + "hosts: !custom [a, b]\n" + "plain: value\n", + encoding="utf-8", + ) + + def test_every_local_tag_loads_as_its_raw_value(self): + self.assertEqual( + { + "motd": TaggedValue("!unsafe", "{{ literal }}"), + "secret": TaggedValue("!vault", "$ANSIBLE_VAULT;1.1;AES256\n6162\n"), + "hosts": TaggedValue("!custom", ["a", "b"]), + "plain": "value", + }, + load_yaml_tagged(self.path), + ) + + def test_load_yaml_any_still_reads_vault_as_its_ciphertext(self): + self.path.write_text("secret: !vault abc\n", encoding="utf-8") + load_yaml_tagged(self.path) + self.assertEqual({"secret": "abc"}, load_yaml_any(self.path)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/python/utils/inventory/test_groups.py b/tests/unit/python/utils/inventory/test_groups.py index 76a79270b6..4a6db23e5a 100644 --- a/tests/unit/python/utils/inventory/test_groups.py +++ b/tests/unit/python/utils/inventory/test_groups.py @@ -1,4 +1,4 @@ -"""Unit tests for utils.inventory.groups.inventory_has_group.""" +"""Unit tests for utils.inventory.groups.""" from __future__ import annotations @@ -7,7 +7,92 @@ import unittest from pathlib import Path from tempfile import TemporaryDirectory -from utils.inventory.groups import inventory_has_group +from utils.inventory.groups import inventory_groups, inventory_has_group + + +class TestInventoryGroups(unittest.TestCase): + def setUp(self) -> None: + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + + def _write(self, name: str, body: str) -> str: + path = self.root / name + path.write_text(textwrap.dedent(body).lstrip("\n"), encoding="utf-8") + return str(path) + + def test_yaml_lists_top_level_and_nested_children(self) -> None: + path = self._write( + "inv.yml", + """ + all: + children: + web-app-cst-store: + children: + nested-group: + hosts: {} + hosts: + localhost: {} + """, + ) + self.assertEqual( + {"all", "web-app-cst-store", "nested-group"}, inventory_groups(path) + ) + + def test_yaml_hosts_and_vars_are_no_groups(self) -> None: + path = self._write( + "inv.yml", + """ + all: + hosts: + web-app-cst-host: {} + vars: + applications: + web-app-cst-store: {custom: true} + """, + ) + self.assertEqual({"all"}, inventory_groups(path)) + + def test_ini_lists_sections_and_children(self) -> None: + path = self._write( + "hosts", + """ + # [comment-group] + [web-app-cst-store] + localhost + + [parent:children] + child-group + + [parent:vars] + key=value + """, + ) + self.assertEqual( + {"web-app-cst-store", "parent", "child-group"}, inventory_groups(path) + ) + + def test_yaml_with_ansible_tags_lists_its_groups(self) -> None: + path = self._write( + "inv.yml", + """ + all: + hosts: + localhost: + motd: !unsafe '{{ literal }}' + secret: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 6162 + children: + web-app-cst-store: + hosts: {} + """, + ) + self.assertEqual({"all", "web-app-cst-store"}, inventory_groups(path)) + + def test_missing_yaml_inventory_raises(self) -> None: + with self.assertRaises(FileNotFoundError): + inventory_groups(str(self.root / "absent.yml")) class TestInventoryHasGroupYaml(unittest.TestCase): diff --git a/tests/unit/python/utils/roles/test_custom.py b/tests/unit/python/utils/roles/test_custom.py new file mode 100644 index 0000000000..1a3dc96ae9 --- /dev/null +++ b/tests/unit/python/utils/roles/test_custom.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import unittest + +from utils.roles.custom import is_custom, is_custom_path +from utils.roles.mapping import ROLE_FILE_TASKS_MAIN + + +class TestIsCustom(unittest.TestCase): + def test_reserved_prefix_is_custom(self): + self.assertTrue(is_custom("web-app-cst-security-check")) + + def test_in_tree_application_is_not_custom(self): + self.assertFalse(is_custom("web-app-nextcloud")) + + def test_category_name_alone_is_not_custom(self): + self.assertFalse(is_custom("web-app-cst")) + + +class TestIsCustomPath(unittest.TestCase): + def test_file_in_a_placed_custom_role(self): + self.assertTrue( + is_custom_path(f"roles/web-app-cst-store/{ROLE_FILE_TASKS_MAIN}") + ) + + def test_file_in_an_in_tree_role(self): + self.assertFalse( + is_custom_path(f"roles/web-app-nextcloud/{ROLE_FILE_TASKS_MAIN}") + ) + + def test_custom_prefix_outside_roles(self): + self.assertFalse(is_custom_path("docs/web-app-cst-store/README.md")) + + def test_roles_directory_itself(self): + self.assertFalse(is_custom_path("roles")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/tests/swarm/test_matrix.py b/tests/unit/python/utils/tests/swarm/test_matrix.py new file mode 100644 index 0000000000..aa5db995bc --- /dev/null +++ b/tests/unit/python/utils/tests/swarm/test_matrix.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import unittest +from unittest import mock + +from utils.tests.swarm import matrix + + +class TestSwarmDeployCommands(unittest.TestCase): + @mock.patch.object(matrix, "run_step", return_value=0) + def test_round_deploy_keeps_the_placed_custom_roles(self, run_step): + matrix._deploy( + app_id="web-app-cst-demo", + inv_dir="/tmp/inv-0", + extras_path="/tmp/extras.yml", + round_index=0, + total=1, + ) + cmd = run_step.call_args.kwargs["cmd"] + self.assertIn("cli.administration.deploy.swarm", cmd) + self.assertIn("--skip-pull", cmd) + + @mock.patch.object(matrix, "run_step", return_value=0) + def test_backup_host_deploy_keeps_the_placed_custom_roles(self, run_step): + matrix._deploy_backup_host( + app_id="web-app-cst-demo", + inv_dir="/tmp/inv-0", + extras_path="/tmp/extras.yml", + ) + cmd = run_step.call_args.kwargs["cmd"] + self.assertIn("cli.administration.deploy.dedicated", cmd) + self.assertIn("--skip-pull", cmd) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/cache/files.py b/utils/cache/files.py index 8f310c21e7..68a2d2c497 100644 --- a/utils/cache/files.py +++ b/utils/cache/files.py @@ -29,10 +29,13 @@ view without subprocessing. from __future__ import annotations import os +import subprocess from functools import lru_cache from pathlib import Path from typing import TYPE_CHECKING +from utils.roles.custom import is_custom_path + from . import PROJECT_ROOT from .gitignore import is_path_gitignored, load_gitignore_patterns @@ -196,6 +199,54 @@ def iter_non_ignored_files( yield path +@lru_cache(maxsize=2) +def repository_files(*, untracked: bool = False) -> tuple[str, ...]: + """Repository-relative POSIX paths of every file ``git ls-files`` lists. + + Git ignores the custom roles an inventory pulls into ``roles/``, yet + every scan checks them like in-tree roles, so their files are listed + as well. + + Args: + untracked: also list the untracked files ``.gitignore`` does not exclude. + + Raises: + OSError: git is not installed. + subprocess.CalledProcessError: git cannot list the files. + """ + root = str(PROJECT_ROOT) + rels = _git_files() + if untracked: + rels.update(untracked_files()) + rels.update( + rel + for path in iter_project_files() + if is_custom_path(rel := os.path.relpath(path, root).replace(os.sep, "/")) + ) + return tuple(sorted(rels)) + + +@lru_cache(maxsize=1) +def untracked_files() -> tuple[str, ...]: + """Repository-relative POSIX paths of the untracked files ``.gitignore`` does not exclude. + + Raises: + OSError: git is not installed. + subprocess.CalledProcessError: git cannot list the files. + """ + return tuple(sorted(_git_files("--others", "--exclude-standard"))) + + +def _git_files(*options: str) -> set[str]: + out = subprocess.run( + ["git", "-c", "safe.directory=*", "ls-files", "-z", *options], + cwd=str(PROJECT_ROOT), + capture_output=True, + check=True, + ).stdout + return {rel for rel in out.decode("utf-8", errors="replace").split("\0") if rel} + + def _reset() -> None: """Test-only helper: clear the project-walk and file-content caches. @@ -206,3 +257,5 @@ def _reset() -> None: _all_project_files.cache_clear() _local_exclude_dirs.cache_clear() read_text.cache_clear() + repository_files.cache_clear() + untracked_files.cache_clear() diff --git a/utils/cache/gitignore.py b/utils/cache/gitignore.py index 3503f8b7bf..ea5355ae62 100644 --- a/utils/cache/gitignore.py +++ b/utils/cache/gitignore.py @@ -26,6 +26,8 @@ from functools import lru_cache from pathlib import Path from typing import TYPE_CHECKING +from utils.roles.custom import ROLES_GLOB + from . import PROJECT_ROOT if TYPE_CHECKING: @@ -45,6 +47,10 @@ def load_gitignore_patterns(root: str | None = None) -> tuple[str, ...]: would require a real pathspec implementation. If a future ``.gitignore`` adds them, extend this helper rather than duplicating the logic in callers. + + The custom-role pattern (:data:`utils.roles.custom.ROLES_GLOB`) is + left out: Git ignores pulled custom roles, yet every project-tree + scan checks them like in-tree roles. """ base = Path(root) if root else PROJECT_ROOT gitignore = base / ".gitignore" @@ -53,7 +59,7 @@ def load_gitignore_patterns(root: str | None = None) -> tuple[str, ...]: patterns: list[str] = [] for raw in gitignore.read_text(encoding="utf-8").splitlines(): line = raw.strip() - if not line or line.startswith(("#", "!")): + if not line or line.startswith(("#", "!")) or line == ROLES_GLOB: continue patterns.append(line) return tuple(patterns) diff --git a/utils/cache/yaml.py b/utils/cache/yaml.py index 7666265ec6..40a82fbd9d 100644 --- a/utils/cache/yaml.py +++ b/utils/cache/yaml.py @@ -32,6 +32,7 @@ CACHE SEMANTICS from __future__ import annotations from collections.abc import Mapping +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -61,8 +62,42 @@ def _construct_vault_tag(loader, node): yaml.SafeLoader.add_constructor("!vault", _construct_vault_tag) _Loader.add_constructor("!vault", _construct_vault_tag) + +@dataclass(frozen=True) +class TaggedValue: + """A value under a local YAML tag, such as Ansible's ``!unsafe`` or ``!vault``, kept as written. + + Args: + tag: the tag, for example ``!vault``. + value: the scalar text, list or mapping under the tag; never rendered or decrypted. + """ + + tag: str + value: Any + + +def _construct_tagged(loader, _suffix, node): + if isinstance(node, yaml.SequenceNode): + return TaggedValue(node.tag, loader.construct_sequence(node, deep=True)) + if isinstance(node, yaml.MappingNode): + return TaggedValue(node.tag, loader.construct_mapping(node, deep=True)) + return TaggedValue(node.tag, loader.construct_scalar(node)) + + +class _TaggedLoader(_Loader): + pass + + +_TaggedLoader.yaml_constructors = { + tag: constructor + for tag, constructor in _Loader.yaml_constructors.items() + if not (tag or "").startswith("!") +} +_TaggedLoader.add_multi_constructor("!", _construct_tagged) + _MISSING = object() _CACHE: dict[tuple[str, int, int], list[Any]] = {} +_TAGGED_CACHE: dict[tuple[str, int, int], list[Any]] = {} def _path_key(path) -> str: @@ -85,18 +120,21 @@ def _load_docs(path) -> list[Any]: synthetic ``default_if_missing`` value never enters the cache and a later ``dump_yaml`` is picked up cleanly). """ - p = Path(path) + return _cached_docs(Path(path), _Loader, _CACHE) + + +def _cached_docs(p: Path, loader, cache: dict) -> list[Any]: sig = _signature(p) - if sig in _CACHE: - return _CACHE[sig] + if sig in cache: + return cache[sig] path_str = sig[0] - for stale_key in [k for k in _CACHE if k[0] == path_str and k != sig]: - _CACHE.pop(stale_key, None) + for stale_key in [k for k in cache if k[0] == path_str and k != sig]: + cache.pop(stale_key, None) with p.open("r", encoding="utf-8") as f: - docs = list(yaml.load_all(f, Loader=_Loader)) # nocheck: direct-yaml - _CACHE[sig] = docs + docs = list(yaml.load_all(f, Loader=loader)) # nocheck: direct-yaml + cache[sig] = docs return docs @@ -114,7 +152,24 @@ def _load_raw(path, *, default_if_missing: Any) -> Any: raise FileNotFoundError(p) return default_if_missing - docs = _load_docs(p) + return _single_doc(path, _load_docs(p)) + + +def load_yaml_tagged(path) -> Any: + """Load a single-document YAML file, keeping every local tag as a :class:`TaggedValue`. + + Reads files Ansible reads, where ``!unsafe`` and ``!vault`` are + legal, without rendering or decrypting any value. The parse is + cached apart from :func:`load_yaml_any`, which loads ``!vault`` as + its ciphertext string. + + Args: + path: the YAML file; a missing one raises ``FileNotFoundError``. + """ + return _single_doc(path, _cached_docs(Path(path), _TaggedLoader, _TAGGED_CACHE)) + + +def _single_doc(path, docs: list[Any]) -> Any: if not docs: return {} if len(docs) > 1: @@ -250,8 +305,9 @@ def load_yaml_all_str(text: str) -> list: def _drop_path(path) -> None: """Drop every cache entry for `path`.""" path_str = _path_key(path) - for stale_key in [k for k in _CACHE if k[0] == path_str]: - _CACHE.pop(stale_key, None) + for cache in (_CACHE, _TAGGED_CACHE): + for stale_key in [k for k in cache if k[0] == path_str]: + cache.pop(stale_key, None) def invalidate(path) -> None: @@ -275,6 +331,7 @@ def _reset() -> None: orchestrates calls to all of them. """ _CACHE.clear() + _TAGGED_CACHE.clear() _reset_cache_for_tests = _reset diff --git a/utils/env/handlers/passthrough.py b/utils/env/handlers/passthrough.py index 07f93e2da0..2c01ff2c5b 100644 --- a/utils/env/handlers/passthrough.py +++ b/utils/env/handlers/passthrough.py @@ -83,6 +83,7 @@ STATIC_KEYS = ( "INFINITO_PLAYWRIGHT_KEEP", "INFINITO_INVENTORY_EXTRA_VARS", "INFINITO_PUBLISH_PORTS", + "INFINITO_EXTRA_MOUNTS", "INFINITO_SKILLS_REPOSITORY", "INFINITO_SKIP_COMPILE", "INFINITO_MAIL_PRODUCTION_HOST", diff --git a/utils/inventory/groups.py b/utils/inventory/groups.py index b0ec8e445b..10c83eafbd 100644 --- a/utils/inventory/groups.py +++ b/utils/inventory/groups.py @@ -1,10 +1,10 @@ -"""Static-inventory group-existence checks (YAML + INI).""" +"""Static-inventory group lookups (YAML + INI).""" from __future__ import annotations from pathlib import Path -from utils.cache.yaml import load_yaml_any +from utils.cache.yaml import load_yaml_any, load_yaml_tagged def inventory_has_group(inventory_path: str, group_name: str) -> bool: @@ -14,6 +14,48 @@ def inventory_has_group(inventory_path: str, group_name: str) -> bool: return _find_ini_section(inventory_path, group_name) +def inventory_groups(inventory_path: str) -> set[str]: + """Return every group a static inventory file declares. + + Ansible tags such as ``!unsafe`` and ``!vault`` load as opaque values. + + Args: + inventory_path: YAML (``.yml``/``.yaml``) or INI inventory file. + """ + if Path(inventory_path).suffix in (".yml", ".yaml"): + return _yaml_groups(load_yaml_tagged(inventory_path)) + return _ini_groups(inventory_path) + + +def _yaml_groups(node) -> set[str]: + groups: set[str] = set() + if not isinstance(node, dict): + return groups + for name, body in node.items(): + groups.add(str(name)) + if isinstance(body, dict): + groups |= _yaml_groups(body.get("children")) + return groups + + +def _ini_groups(inventory_path: str) -> set[str]: + groups: set[str] = set() + in_children = False + with Path(inventory_path).open(encoding="utf-8") as f: + for raw in f: + line = raw.strip() + if not line or line.startswith(("#", ";")): + continue + if line.startswith("[") and line.endswith("]"): + name, _, kind = line[1:-1].strip().partition(":") + groups.add(name) + in_children = kind == "children" + continue + if in_children: + groups.add(line.split()[0]) + return groups + + def _find_yaml_key(node, key: str) -> bool: if isinstance(node, dict): for k, v in node.items(): diff --git a/utils/roles/custom.py b/utils/roles/custom.py new file mode 100644 index 0000000000..5a8ee0b5ce --- /dev/null +++ b/utils/roles/custom.py @@ -0,0 +1,40 @@ +"""Custom-role naming SPOT. + +A custom role is the inventory group ``web-app-cst-``. Its +``applications.`` block marks it with ``custom: true`` and names +the repository it comes from under ``source``. The pull places it as +``roles/``, which ``.gitignore`` keeps out of every core commit. + +Keep this module free of imports beyond the standard library: the +project-tree scan helpers in :mod:`utils.cache` read it. +""" + +from __future__ import annotations + +PREFIX: str = "web-app-cst-" +ROLES_GLOB: str = f"roles/{PREFIX}*" +CUSTOM_KEY: str = "custom" +SOURCE_KEY: str = "source" + + +class CustomRoleError(Exception): + """A custom-role declaration, fetch or placement that stops the run.""" + + +def is_custom(name: str) -> bool: + """Whether *name* carries the reserved custom-role prefix. + + Args: + name: a role directory, inventory group or application_id. + """ + return name.startswith(PREFIX) + + +def is_custom_path(rel_path: str) -> bool: + """Whether a repository-relative POSIX path lies in a placed custom role. + + Args: + rel_path: path relative to the repository root, ``/``-separated. + """ + parts = rel_path.split("/") + return len(parts) > 1 and parts[0] == "roles" and is_custom(parts[1]) diff --git a/utils/tests/swarm/matrix.py b/utils/tests/swarm/matrix.py index f6d2302a30..73de8d3d2b 100644 --- a/utils/tests/swarm/matrix.py +++ b/utils/tests/swarm/matrix.py @@ -136,6 +136,7 @@ def _deploy( "-p", f"{inv_dir}/.password", "--skip-build", + "--skip-pull", "--skip-cleanup", "--skip-backup", "-e", @@ -172,6 +173,7 @@ def _deploy_backup_host(*, app_id: str, inv_dir: str, extras_path: str) -> int: "-p", f"{inv_dir}/.password", "--skip-build", + "--skip-pull", "--skip-cleanup", "--skip-backup", "-e",