From ebd7d2e72be3287f020512d5bdccd3bb79ea75e1 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 28 Jul 2026 11:19:30 +0200 Subject: [PATCH] refactor(meta): centralize category/distro SPOTs categories.yml moves from roles/ to the repository-root meta/, joining packages.yml. utils/roles/categories.py owns its location plus the tree-walking helpers that were stranded in entity/name.py, so no caller spells the path or duplicates the lookup. meta/distros.yml becomes the SPOT for every distribution: os_family, Galaxy platform name, dev-environment base image, and the pkgmgr/pkgmgr-virgin/environment reference templates. It replaces seven scattered copies (DISTRO_FAMILY, VALID_DISTROS, _DISTROS, _CANONICAL_PLATFORMS, two shell arrays, a Makefile literal). INFINITO_DISTROS is no longer a static default but generated by utils/env/handlers/infinito/distros.py; the shell consumers read it through the env layer. Drops dead CLI surface found along the way: --categories-file and --roles-dir on the provision CLI plus three unused parameters of generate_dynamic_inventory, and the positional roles_file of the invokable categories command. tests/lint/repository/test_distros_spot.py pins the copies that cannot call the SPOT (workflow literals, docs table) and forbids composing a pkgmgr reference outside it. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/test-guide.yml | 6 +- Makefile | 2 +- cli/administration/deploy/development/env.py | 6 +- cli/administration/inventory/provision/cli.py | 37 +---- .../provision/inventory_generator.py | 3 - cli/meta/categories/README.md | 2 +- cli/meta/categories/invokable/command.py | 24 +-- cli/meta/ci/guide_select.py | 28 +++- cli/meta/ci/image_ref.py | 59 +++++++ cli/meta/roles/order/run/__main__.py | 4 +- cli/meta/roles/services/called/__init__.py | 33 ++-- default.env | 3 - {roles => meta}/categories.yml | 0 meta/distros.yml | 59 +++++++ plugins/filter/invokable_paths.py | 4 +- plugins/lookup/stage_groups.py | 2 +- scripts/github/resolve/pick_distro.sh | 13 +- scripts/tests/install/make.sh | 8 +- scripts/tests/install/pkgmgr.sh | 11 +- tests/integration/meta/categories/README.md | 2 +- .../meta/categories/test_invokable_parents.py | 9 +- .../integration/meta/categories/test_paths.py | 3 +- .../roles/applications/id/test_basic.py | 6 +- .../roles/meta/test_main_galaxy_schema.py | 15 +- .../roles/test_compose_resource_limits.py | 2 +- tests/lint/repository/test_distros_spot.py | 154 ++++++++++++++++++ .../provision/test_inventory_generator.py | 3 - .../meta/categories/invokable/test_command.py | 19 ++- .../filter/get/test_all_invokable_apps.py | 3 +- .../plugins/filter/get/test_entity_name.py | 13 +- tests/unit/utils/cleanup/test_nginx_vhosts.py | 17 +- tests/unit/utils/roles/test_entity_apps.py | 11 +- tests/unit/utils/roles/test_stage.py | 2 +- .../utils/roles/validation/test_invokable.py | 30 ++-- tests/unit/utils/test_distros.py | 70 ++++++++ utils/distros.py | 117 +++++++++++++ utils/env/handlers/__init__.py | 2 + utils/env/handlers/infinito/distros.py | 21 +++ utils/env/handlers/infinito/image.py | 4 +- utils/env/handlers/infinito/parent_image.py | 4 +- utils/env/handlers/passthrough.py | 1 - utils/packages/schema.py | 13 +- utils/roles/categories.py | 75 +++++++++ utils/roles/entity/name.py | 41 +---- utils/roles/required_by_coverage.py | 5 +- utils/roles/stage.py | 19 +-- utils/roles/validation/invokable.py | 33 ++-- 47 files changed, 740 insertions(+), 258 deletions(-) create mode 100644 cli/meta/ci/image_ref.py rename {roles => meta}/categories.yml (100%) create mode 100644 meta/distros.yml create mode 100644 tests/lint/repository/test_distros_spot.py create mode 100644 tests/unit/utils/test_distros.py create mode 100644 utils/distros.py create mode 100644 utils/env/handlers/infinito/distros.py create mode 100644 utils/roles/categories.py diff --git a/.github/workflows/test-guide.yml b/.github/workflows/test-guide.yml index 92955f6685..2dbd76d354 100644 --- a/.github/workflows/test-guide.yml +++ b/.github/workflows/test-guide.yml @@ -90,7 +90,11 @@ jobs: shell: bash run: | set -euo pipefail - INFINITO_IMAGE="ghcr.io/$(scripts/meta/resolve/repository/owner.sh)/$(scripts/meta/resolve/repository/name.sh)/${INFINITO_DISTRO}:ci-${{ github.sha }}" + INFINITO_IMAGE="$(python3 -m cli.meta.ci.image_ref --kind environment \ + --distro "${INFINITO_DISTRO}" \ + --owner "$(scripts/meta/resolve/repository/owner.sh)" \ + --repository "$(scripts/meta/resolve/repository/name.sh)" \ + --tag "ci-${{ github.sha }}")" echo "INFINITO_IMAGE=${INFINITO_IMAGE}" >> "$GITHUB_ENV" export INFINITO_IMAGE make compose-up diff --git a/Makefile b/Makefile index 9ac6d6ca72..34b9dd187e 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ build-cleanup: .PHONY: build-dependency # Pull the build dependency image. build-dependency: - @docker pull ghcr.io/kevinveenbirkenbach/pkgmgr-$${INFINITO_DISTRO}:stable + @docker pull "$${INFINITO_PARENT_IMAGE:?Run 'make dotenv' to generate the .env single source of truth}" .PHONY: build-missing # Build the local image if it is missing. diff --git a/cli/administration/deploy/development/env.py b/cli/administration/deploy/development/env.py index 251069da34..842f6d3bba 100644 --- a/cli/administration/deploy/development/env.py +++ b/cli/administration/deploy/development/env.py @@ -2,7 +2,9 @@ from __future__ import annotations import os -VALID_DISTROS: tuple[str, ...] = ("arch", "debian", "ubuntu", "fedora", "centos") +from utils.distros import distro_names + +VALID_DISTROS: tuple[str, ...] = distro_names() def compose_file_args() -> list[str]: @@ -23,7 +25,7 @@ def resolve_distro() -> str: if not distro: raise SystemExit( "INFINITO_DISTRO is not set. Run 'make dotenv' (or source scripts/meta/env/load.sh) " - "or export INFINITO_DISTRO= " + f"or export INFINITO_DISTRO=<{'|'.join(VALID_DISTROS)}> " "before invoking cli.administration.deploy.development." ) if distro not in VALID_DISTROS: diff --git a/cli/administration/inventory/provision/cli.py b/cli/administration/inventory/provision/cli.py index 4c9831018d..44232e2f5f 100644 --- a/cli/administration/inventory/provision/cli.py +++ b/cli/administration/inventory/provision/cli.py @@ -35,20 +35,6 @@ def _resolve_inventory_file( return (inventory_dir / "devices.yml").resolve() -def _resolve_roles_dir(project_root: Path, roles_dir_arg: str | None) -> Path: - return ( - Path(roles_dir_arg) if roles_dir_arg else (project_root / "roles") - ).resolve() - - -def _resolve_categories_file(roles_dir: Path, categories_file_arg: str | None) -> Path: - return ( - Path(categories_file_arg) - if categories_file_arg - else (roles_dir / "categories.yml") - ).resolve() - - def _resolve_mirrors_file(project_root: Path, mirror_arg: str | None) -> Path | None: """ --mirror can be used in two ways: @@ -60,7 +46,6 @@ def _resolve_mirrors_file(project_root: Path, mirror_arg: str | None) -> Path | if mirror_arg is None: return None - # argparse uses const="mirrors.yml" when flag is present without value candidate = Path(mirror_arg) if not candidate.is_absolute(): candidate = (project_root / candidate).resolve() @@ -121,14 +106,6 @@ def main(argv: list[str] | None = None) -> int: default=None, help="Vault password file path. Default: /.password", ) - parser.add_argument( - "--roles-dir", default=None, help="Path to roles/ (default: /roles)." - ) - parser.add_argument( - "--categories-file", - default=None, - help="Path to roles/categories.yml (default: /categories.yml).", - ) parser.add_argument( "--workers", type=int, @@ -144,7 +121,6 @@ def main(argv: list[str] | None = None) -> int: ), ) - # New: mirroring support parser.add_argument( "--mirror", nargs="?", @@ -182,8 +158,7 @@ def main(argv: list[str] | None = None) -> int: project_root = detect_project_root(Path(__file__).resolve()) env = build_env_with_project_root(project_root) - roles_dir = _resolve_roles_dir(project_root, args.roles_dir) - categories_file = _resolve_categories_file(roles_dir, args.categories_file) + roles_dir = (project_root / "roles").resolve() mirrors_file = _resolve_mirrors_file(project_root, args.mirror) inventory_dir = Path(args.inventory_dir).resolve() @@ -192,7 +167,6 @@ def main(argv: list[str] | None = None) -> int: inventory_file = _resolve_inventory_file(inventory_dir, args.inventory_file) host_vars_file = (inventory_dir / "host_vars" / f"{args.host}.yml").resolve() - # Vault password file if args.vault_password_file: vault_password_file = Path(args.vault_password_file).resolve() else: @@ -225,10 +199,7 @@ def main(argv: list[str] | None = None) -> int: ) dyn_inv = generate_dynamic_inventory( host=args.host, - roles_dir=roles_dir, - categories_file=categories_file, tmp_inventory=tmp_inventory, - project_root=project_root, env=env, ) @@ -242,7 +213,6 @@ def main(argv: list[str] | None = None) -> int: dyn_children = (dyn_inv.get("all", {}) or {}).get("children", {}) or {} application_ids = sorted(dyn_children.keys()) - # Merge inventory file if inventory_file.exists(): print(f"[INFO] Merging into existing inventory: {inventory_file}") base_inv = load_yaml(inventory_file) @@ -253,7 +223,6 @@ def main(argv: list[str] | None = None) -> int: merged_inv = _merge_inventories(base_inv, dyn_inv, host=args.host) dump_yaml(inventory_file, merged_inv) - # Host vars print(f"[INFO] Ensuring host_vars for host '{args.host}' at {host_vars_file}") ensure_host_vars_file( host_vars_file=host_vars_file, @@ -283,7 +252,6 @@ def main(argv: list[str] | None = None) -> int: f"--app-variants[{app_id!r}] must be an integer, got {raw_index!r}: {exc}" ) - # Credentials if application_ids: print( f"[INFO] Generating credentials for {len(application_ids)} applications..." @@ -317,13 +285,10 @@ def main(argv: list[str] | None = None) -> int: ) apply_vars_overrides(host_vars_file=host_vars_file, json_str=args.vars) - # Mirror overrides should win (requested behavior: "überschreiben") if mirrors_file is not None: print(f"[INFO] Applying mirror overrides from: {mirrors_file}") apply_mirror_overrides(host_vars_file=host_vars_file, mirrors_file=mirrors_file) - # Disable services listed in the `disable` env var (space- or comma-separated). - # Also removes the provider roles from the inventory. apply_services_disabled_from_env( host_vars_file=host_vars_file, inventory_file=inventory_file, diff --git a/cli/administration/inventory/provision/inventory_generator.py b/cli/administration/inventory/provision/inventory_generator.py index ddb3fcdf38..26a50d5bcc 100644 --- a/cli/administration/inventory/provision/inventory_generator.py +++ b/cli/administration/inventory/provision/inventory_generator.py @@ -12,10 +12,7 @@ if TYPE_CHECKING: def generate_dynamic_inventory( host: str, - roles_dir: Path, - categories_file: Path, tmp_inventory: Path, - project_root: Path, env: dict[str, str] | None, ) -> dict[str, Any]: """ diff --git a/cli/meta/categories/README.md b/cli/meta/categories/README.md index a3568d9d18..fc88531590 100644 --- a/cli/meta/categories/README.md +++ b/cli/meta/categories/README.md @@ -1,3 +1,3 @@ # Categories 🗂️ -Inspect role categories declared under `roles/categories.yml`, including the set of paths that the dispatcher treats as invokable. +Inspect role categories declared under `meta/categories.yml`, including the set of paths that the dispatcher treats as invokable. diff --git a/cli/meta/categories/invokable/command.py b/cli/meta/categories/invokable/command.py index ea888841f2..506ce4c616 100755 --- a/cli/meta/categories/invokable/command.py +++ b/cli/meta/categories/invokable/command.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 """ -CLI for extracting invokable or non-invokable role paths from a nested roles YAML file. +CLI for extracting invokable or non-invokable role paths from the category SPOT. Layout assumption: this module sits under ``cli/`` and reads -``/roles/categories.yml`` plus filter plugins under +``/meta/categories.yml`` plus filter plugins under ``/plugins/filter/``. + +The two ``plugins.filter.invokable_paths`` imports are bound at module +level so ``unittest.mock.patch`` can replace them in tests. """ from __future__ import annotations @@ -14,9 +17,6 @@ import sys import yaml -# IMPORTANT: -# These imports must exist at module level so unittest.mock.patch() -# can replace them in tests. from plugins.filter.invokable_paths import ( get_invokable_paths, get_non_invokable_paths, @@ -27,14 +27,7 @@ from . import PROJECT_ROOT def main() -> None: parser = argparse.ArgumentParser( - description="Extract invokable or non-invokable role paths from a nested roles YAML file." - ) - - parser.add_argument( - "roles_file", - nargs="?", - default=None, - help="Path to roles/categories.yml (optional).", + description="Extract invokable or non-invokable role paths from the category SPOT." ) parser.add_argument( @@ -60,15 +53,14 @@ def main() -> None: args = parser.parse_args() - # Ensure repo root is importable (CLI use outside repo root) repo_root = PROJECT_ROOT sys.path.insert(0, str(repo_root)) try: if args.non_invokable: - paths = get_non_invokable_paths(args.roles_file, args.suffix) + paths = get_non_invokable_paths(suffix=args.suffix) else: - paths = get_invokable_paths(args.roles_file, args.suffix) + paths = get_invokable_paths(suffix=args.suffix) except FileNotFoundError as e: print(f"Error: {e}", file=sys.stderr) diff --git a/cli/meta/ci/guide_select.py b/cli/meta/ci/guide_select.py index 9c244080c4..cc80528a72 100644 --- a/cli/meta/ci/guide_select.py +++ b/cli/meta/ci/guide_select.py @@ -20,6 +20,8 @@ import sys from typing import TYPE_CHECKING from utils.cache.files import PROJECT_ROOT +from utils.distros import distro_names, pkgmgr_image +from utils.env.parser import parse_static_env from utils.roles.deploy import role_has_stack from utils.roles.lifecycle import tested_lifecycles from utils.roles.meta_lookup import get_role_test_skips @@ -32,15 +34,23 @@ from utils.roles.validation.invokable import ( if TYPE_CHECKING: from pathlib import Path -_RUNTIME_IMAGE = "ghcr.io/kevinveenbirkenbach/pkgmgr-{distro}:stable" - -_DISTROS = ("arch", "debian", "ubuntu", "fedora", "centos") - def _tokens(raw: str) -> list[str]: return [t for t in raw.split() if t] +def _static_env(key: str) -> str: + """Env value, falling back to the ``default.env`` declaration. + + The Guide workflow exports only a handful of keys explicitly; the rest + are read from the static SPOT rather than hardcoded here. + """ + value = os.environ.get(key) + if value: + return value + return parse_static_env(PROJECT_ROOT / "default.env")[key] + + def _guide_mode(role_dir: Path) -> str: return "compose" if role_has_stack(role_dir) else "host" @@ -75,7 +85,8 @@ def main(argv: list[str] | None = None) -> int: return 1 role = random.choice(pool) # noqa: S311 - test-role pick, not cryptographic - distros = [d for d in _tokens(os.environ["INFINITO_DISTROS"]) if d in _DISTROS] + known = distro_names() + distros = [d for d in _tokens(os.environ["INFINITO_DISTROS"]) if d in known] if not distros: print("guide_select: no known distro in INFINITO_DISTROS", file=sys.stderr) return 1 @@ -86,7 +97,12 @@ def main(argv: list[str] | None = None) -> int: print(f"GUIDE_ROLE={role}") print(f"GUIDE_MODE={mode}") print(f"INFINITO_DISTRO={distro}") - print(f"GUIDE_RUNTIME_IMAGE={_RUNTIME_IMAGE.format(distro=distro)}") + runtime_image = pkgmgr_image( + distro, + owner=_static_env("INFINITO_PARENT_IMAGE_OWNER"), + tag=_static_env("INFINITO_PARENT_IMAGE_TAG"), + ) + print(f"GUIDE_RUNTIME_IMAGE={runtime_image}") return 0 diff --git a/cli/meta/ci/image_ref.py b/cli/meta/ci/image_ref.py new file mode 100644 index 0000000000..a60a24d1f0 --- /dev/null +++ b/cli/meta/ci/image_ref.py @@ -0,0 +1,59 @@ +"""Render a distro image reference from the ``meta/distros.yml`` SPOT. + +Exists so shell and workflow steps can compose the same reference the env +layer composes, instead of spelling ``ghcr.io/...`` out a second time. + + python3 -m cli.meta.ci.image_ref --kind environment \\ + --distro debian --owner acme --repository infinito-nexus --tag ci-abc +""" + +from __future__ import annotations + +import argparse +import sys + +from utils.distros import ( + IMAGE_ENVIRONMENT, + IMAGE_PKGMGR, + environment_image, + pkgmgr_image, +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--kind", + required=True, + choices=(IMAGE_PKGMGR, IMAGE_ENVIRONMENT), + help="Which image reference to render.", + ) + parser.add_argument("--distro", required=True, help="Distro id, e.g. debian.") + parser.add_argument("--owner", required=True, help="Registry owner.") + parser.add_argument("--tag", required=True, help="Image tag.") + parser.add_argument( + "--repository", + default="", + help="Repository name; required for --kind environment.", + ) + args = parser.parse_args(argv) + + if args.kind == IMAGE_PKGMGR: + print(pkgmgr_image(args.distro, owner=args.owner, tag=args.tag)) + return 0 + + if not args.repository: + parser.error("--repository is required for --kind environment") + print( + environment_image( + args.distro, + owner=args.owner, + repository=args.repository, + tag=args.tag, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/cli/meta/roles/order/run/__main__.py b/cli/meta/roles/order/run/__main__.py index 1fe500f618..8a2a6c9a59 100755 --- a/cli/meta/roles/order/run/__main__.py +++ b/cli/meta/roles/order/run/__main__.py @@ -8,11 +8,11 @@ Group-agnostic role call order. - 📦 Phase 2 (main): every remaining invokable role that is NOT a preload service, topologically sorted by run_after among themselves. Roles that become ready together fall out in stage order (constructor -> workstation - -> server -> destructor) via ``roles/categories.yml``, then category + -> server -> destructor) via ``meta/categories.yml``, then category run_after, then name. Derived purely from ``meta/*.yml`` (run_after + the service registry) and -``roles/categories.yml`` (stage), NOT from ``tasks/stages/*.yml``, so it +``meta/categories.yml`` (stage), NOT from ``tasks/stages/*.yml``, so it stays independent of how the stage playbooks happen to be split. Examples: diff --git a/cli/meta/roles/services/called/__init__.py b/cli/meta/roles/services/called/__init__.py index 8533d921d8..d13cf1f6b8 100644 --- a/cli/meta/roles/services/called/__init__.py +++ b/cli/meta/roles/services/called/__init__.py @@ -5,7 +5,7 @@ A role's `meta/services.yml` MAY contain (per-service-entity): : required_by: - categories: [web, web.app, ...] # category handles (cf. roles/categories.yml) + categories: [web, web.app, ...] # category handles (cf. meta/categories.yml) roles: [web-app-yourls, ...] # full role names incl. category prefix A `required_by` block MAY instead nest `compose:` / `swarm:` sub-blocks to @@ -25,6 +25,15 @@ non-skip status) in the run log slice that this verifier scans. A missing event indicates a coverage regression in the deploy chain (typically: a once-flag swallowed an include before the role could run). +Two log-parsing trip-wires shape the regexes below. The non-skip status +match is anchored to line start (after optional indent) so a substring in +arbitrary task output cannot fake a status. And Ansible's `log_path` file +(`ANSIBLE_LOG_PATH`, what the swarm/compose deploy feeds this verifier) +prefixes every line with ` p= u= n= | `; left +unstripped that prefix defeats both the line anchor and the `\\nTASK [` +block split, so every required role would false-positive as "did not +execute". + CLI: `python -m cli.meta.roles.services.called --help` """ @@ -40,18 +49,10 @@ from utils.cache.yaml import load_yaml_any from utils.roles.mapping import ROLE_FILE_META_SERVICES _ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") -# Status indicators that prove a task block was NOT skipped. We anchor to -# line start (after optional indent) so substring matches in arbitrary -# task output cannot trigger a false positive. _NON_SKIP_RE = re.compile( r"^\s*(?:ok|changed|fatal|included):\s+", re.MULTILINE, ) -# Ansible's log_path file (ANSIBLE_LOG_PATH, what the swarm/compose deploy feeds -# this verifier) prefixes every line with " p= u= n= | ". -# Unstripped, that prefix defeats both the line-anchored _NON_SKIP_RE and the -# "\nTASK [" block split below (the TASK header no longer follows the newline), -# so every required role false-positives as "did not execute". _LOG_PREFIX_RE = re.compile( r"^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d+ p=\d+ u=\S+ n=\S+ \w+\| ", re.MULTILINE, @@ -79,8 +80,11 @@ _SWARM_MARKER_ROLES = frozenset({"svc-swarm-node", "svc-swarm-manager"}) def _deploy_mode(deployed_role_ids: list[str]) -> str: - # No DEPLOYMENT_MODE in scope post-deploy: infer it from the role set, since a - # swarm deploy always pulls in the swarm-node/manager infra and compose never does. + """Deploy mode inferred from the role set. + + ``DEPLOYMENT_MODE`` is out of scope post-deploy, but a swarm deploy + always pulls in the swarm-node/manager infra and compose never does. + """ return "swarm" if _SWARM_MARKER_ROLES.intersection(deployed_role_ids) else "compose" @@ -120,8 +124,6 @@ def required_role_ids( rb = entry.get("required_by") if not isinstance(rb, dict): continue - # required_by may scope to a deploy mode via 'compose'/'swarm' sub-keys; - # a flat {categories, roles} applies in every mode. if "compose" in rb or "swarm" in rb: rb = rb.get(deploy_mode) or {} if not isinstance(rb, dict): @@ -233,6 +235,9 @@ def verify( Reads from `log_byte_offset` to end and asserts that every required role's body actually executed in that slice (i.e. at least one TASK block for the role ended with a non-skip status). + + An unreadable log proves coverage neither way and is reported as + "missing" for every required role, so the operator notices. """ required = required_role_ids( roles_dir=roles_dir, deployed_role_ids=deployed_role_ids @@ -247,8 +252,6 @@ def verify( else: log_content = _host_log_slice(log_path=log_path, byte_offset=log_byte_offset) if log_content is None: - # Missing log == we cannot prove coverage either way; treat as - # missing for every required role so the operator notices. return False, sorted(required) missing = [ diff --git a/default.env b/default.env index cbfeeb3084..3e1a381608 100644 --- a/default.env +++ b/default.env @@ -106,9 +106,6 @@ INFINITO_DISCOVERY_SORT="desc integrated,asc covered_by,desc embeds,desc consume # Distro the build/test matrix is currently iterating over (generator may override). INFINITO_DISTRO=debian -# Space-separated list of distros the build/test matrix iterates over. -INFINITO_DISTROS="arch debian ubuntu fedora centos" - # Domain the host-side DNS-fallback / dnsmasq config wires up (consumed by scripts/system/network/dns/). INFINITO_DNS_DOMAIN=infinito.example diff --git a/roles/categories.yml b/meta/categories.yml similarity index 100% rename from roles/categories.yml rename to meta/categories.yml diff --git a/meta/distros.yml b/meta/distros.yml new file mode 100644 index 0000000000..7c3bc2af2b --- /dev/null +++ b/meta/distros.yml @@ -0,0 +1,59 @@ +--- +# Distribution SPOT. +# +# Every distribution Infinito.Nexus builds, tests and deploys on is declared +# here exactly once. Consumers read it through utils/distros.py; nothing may +# re-spell a distro list or an image reference locally. +# +# images: reference templates rendered by utils/distros.py. +# {slug} comes from this file +# {owner} {repository} {tag} come from default.env, which stays the SPOT +# for registry owner and tags +# +# distros: the id used as INFINITO_DISTRO, carrying +# slug path segment inside the image reference +# family ansible_facts['os_family'] the distribution reports, +# consumed by the meta/packages.yml registry +# dev_runtime_image upstream base image the development-environment test +# matrix boots the distribution from. Not one of ours, +# so it carries its own registry and tag. +# galaxy_platform platform name Ansible Galaxy knows the distribution by, +# used to build the canonical galaxy_info.platforms list +# every role's meta/main.yml must carry. The versions +# qualifier attached to every such entry is +# galaxy_platform_versions below. + +images: + pkgmgr: "ghcr.io/{owner}/pkgmgr-{slug}:{tag}" + pkgmgr_virgin: "ghcr.io/{owner}/pkgmgr-{slug}-virgin:{tag}" + environment: "ghcr.io/{owner}/{repository}/{slug}:{tag}" + +galaxy_platform_versions: + - all + +distros: + arch: + slug: arch + family: Archlinux + dev_runtime_image: manjarolinux/base + galaxy_platform: ArchLinux + debian: + slug: debian + family: Debian + dev_runtime_image: debian:bookworm + galaxy_platform: Debian + ubuntu: + slug: ubuntu + family: Debian + dev_runtime_image: ubuntu:latest + galaxy_platform: Ubuntu + fedora: + slug: fedora + family: RedHat + dev_runtime_image: fedora:latest + galaxy_platform: Fedora + centos: + slug: centos + family: RedHat + dev_runtime_image: quay.io/centos/centos:latest + galaxy_platform: EL diff --git a/plugins/filter/invokable_paths.py b/plugins/filter/invokable_paths.py index ad3c8d6171..ad9c9bde8b 100644 --- a/plugins/filter/invokable_paths.py +++ b/plugins/filter/invokable_paths.py @@ -2,14 +2,14 @@ from __future__ import annotations import yaml -from utils.cache import ROLES_DIR from utils.cache.yaml import load_yaml_any +from utils.roles.categories import categories_file _METADATA_KEYS = frozenset({"title", "description", "icon", "invokable", "modes"}) def _default_roles_file() -> str: - categories = ROLES_DIR / "categories.yml" + categories = categories_file() if not categories.is_file(): raise FileNotFoundError( f"Could not locate {categories}. Pass roles_file explicitly." diff --git a/plugins/lookup/stage_groups.py b/plugins/lookup/stage_groups.py index 08750af8de..a4d7ebca63 100644 --- a/plugins/lookup/stage_groups.py +++ b/plugins/lookup/stage_groups.py @@ -1,7 +1,7 @@ """Lookup `stage_groups`: the ordered role-group names (``tasks/groups/-roles.yml``) that belong to a deploy stage, in intra-stage call order. The stage plays loop over it so group membership follows -``roles/categories.yml`` instead of a hardcoded list: +``meta/categories.yml`` instead of a hardcoded list: loop: "{{ lookup('stage_groups', 'server') }}" """ diff --git a/scripts/github/resolve/pick_distro.sh b/scripts/github/resolve/pick_distro.sh index 20ec3d9073..740d1bd052 100755 --- a/scripts/github/resolve/pick_distro.sh +++ b/scripts/github/resolve/pick_distro.sh @@ -1,16 +1,21 @@ #!/usr/bin/env bash # # Decide which distros the latest-push CI run should cover. If the -# pushed commit carries a `vX.Y.Z` annotated tag, run every distro in -# `ALL_DISTROS`; otherwise pick one at random. Emits both -# `is_version` and `distros` to GITHUB_OUTPUT. +# pushed commit carries a `vX.Y.Z` annotated tag, run every declared +# distro; otherwise pick one at random. Emits both `is_version` and +# `distros` to GITHUB_OUTPUT. The distro set comes from INFINITO_DISTROS, +# which the env layer generates from the meta/distros.yml SPOT. # # Usage: # pick_distro.sh # (reads GITHUB_SHA from env; writes to GITHUB_OUTPUT) set -euo pipefail -ALL_DISTROS=(arch debian ubuntu fedora centos) +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +# shellcheck source=scripts/meta/env/load.sh +source "${REPO_ROOT}/scripts/meta/env/load.sh" +: "${INFINITO_DISTROS:?INFINITO_DISTROS must be resolved by scripts/meta/env/load.sh}" +read -r -a ALL_DISTROS <<<"${INFINITO_DISTROS}" version_tag="$( git tag --points-at "${GITHUB_SHA}" | diff --git a/scripts/tests/install/make.sh b/scripts/tests/install/make.sh index 9b0830d8d4..9e540b747d 100755 --- a/scripts/tests/install/make.sh +++ b/scripts/tests/install/make.sh @@ -11,6 +11,9 @@ # INFINITO_VENV_DIR venv path inside the container (from default.env). # RUNNER_TEMP runner scratch dir for the per-distro copies. # GITHUB_WORKSPACE the checkout to install and test. +# +# The distro set comes from INFINITO_DISTROS, which the env layer generates +# from the meta/distros.yml SPOT. set -uo pipefail @@ -20,7 +23,10 @@ set -uo pipefail : "${RUNNER_TEMP:?}" : "${GITHUB_WORKSPACE:?}" -distros=(arch debian ubuntu fedora centos) +# shellcheck source=scripts/meta/env/load.sh +source "${GITHUB_WORKSPACE}/scripts/meta/env/load.sh" +: "${INFINITO_DISTROS:?INFINITO_DISTROS must be resolved by scripts/meta/env/load.sh}" +read -r -a distros <<<"${INFINITO_DISTROS}" declare -A pid for d in "${distros[@]}"; do diff --git a/scripts/tests/install/pkgmgr.sh b/scripts/tests/install/pkgmgr.sh index d03452e4d5..07bf65d19b 100755 --- a/scripts/tests/install/pkgmgr.sh +++ b/scripts/tests/install/pkgmgr.sh @@ -10,7 +10,9 @@ # INFINITO_VENV_DIR venv path inside the container (from default.env). # RUNNER_TEMP runner scratch dir for the per-distro copies. # GITHUB_WORKSPACE the checkout to install and test. -# INFINITO_DISTROS distro list; defaults to the SPOT in default.env. +# +# The distro set comes from INFINITO_DISTROS, which the env layer generates +# from the meta/distros.yml SPOT. set -uo pipefail @@ -19,10 +21,9 @@ set -uo pipefail : "${RUNNER_TEMP:?}" : "${GITHUB_WORKSPACE:?}" -if [[ -z "${INFINITO_DISTROS:-}" ]]; then - INFINITO_DISTROS="$(awk -F= '/^INFINITO_DISTROS=/{v=$2; gsub(/^"|"$/,"",v); print v; exit}' "${GITHUB_WORKSPACE}/default.env")" -fi -: "${INFINITO_DISTROS:?}" +# shellcheck source=scripts/meta/env/load.sh +source "${GITHUB_WORKSPACE}/scripts/meta/env/load.sh" +: "${INFINITO_DISTROS:?INFINITO_DISTROS must be resolved by scripts/meta/env/load.sh}" read -r -a distros <<<"${INFINITO_DISTROS}" declare -A pid diff --git a/tests/integration/meta/categories/README.md b/tests/integration/meta/categories/README.md index d001538058..eec8dff906 100644 --- a/tests/integration/meta/categories/README.md +++ b/tests/integration/meta/categories/README.md @@ -1,6 +1,6 @@ # Category Tests 🗂️ -Integration tests that validate `roles/categories.yml`: every declared category path MUST correspond to an existing directory under `roles/`, and `invokable: true` MUST NOT appear on any descendant of an already-invokable ancestor. +Integration tests that validate `meta/categories.yml`: every declared category path MUST correspond to an existing directory under `roles/`, and `invokable: true` MUST NOT appear on any descendant of an already-invokable ancestor. Tests in this directory MUST only cover the category hierarchy itself. Tests for `application_id` classification driven by `invokable_paths.py` MUST live under `tests/integration/roles/applications/id/`. diff --git a/tests/integration/meta/categories/test_invokable_parents.py b/tests/integration/meta/categories/test_invokable_parents.py index e0b930eaf9..92e378a671 100644 --- a/tests/integration/meta/categories/test_invokable_parents.py +++ b/tests/integration/meta/categories/test_invokable_parents.py @@ -1,8 +1,7 @@ import unittest from utils.cache.yaml import load_yaml_any - -from . import PROJECT_ROOT +from utils.roles.categories import categories_file class TestCategoriesInvokableExclusion(unittest.TestCase): @@ -11,7 +10,7 @@ class TestCategoriesInvokableExclusion(unittest.TestCase): Verify that if any ancestor in the hierarchy is invokable, none of its descendants may be invokable. """ - yaml_path = str(PROJECT_ROOT / "roles" / "categories.yml") + yaml_path = str(categories_file()) data = load_yaml_any(yaml_path) @@ -25,7 +24,6 @@ class TestCategoriesInvokableExclusion(unittest.TestCase): is_invokable = value.get("invokable", False) current_path = [*path, key] - # Violation: a descendant is invokable despite an invokable ancestor if ancestor_invokable and is_invokable: ancestor_name = ".".join(path) if path else "" violations.append( @@ -33,15 +31,12 @@ class TestCategoriesInvokableExclusion(unittest.TestCase): f"but its ancestor ({ancestor_name}) is also invokable." ) - # Any_invokable = True if this node or any ancestor is invokable new_ancestor_flag = ancestor_invokable or is_invokable - # Recurse into subcategories for subkey, subval in value.items(): if isinstance(subval, dict): recurse({subkey: subval}, current_path, new_ancestor_flag) - # start at top-level roles, with no invokable ancestor recurse(data.get("roles", {}), [], False) if violations: diff --git a/tests/integration/meta/categories/test_paths.py b/tests/integration/meta/categories/test_paths.py index 91711b6f3d..9842e936f9 100644 --- a/tests/integration/meta/categories/test_paths.py +++ b/tests/integration/meta/categories/test_paths.py @@ -1,6 +1,7 @@ import unittest from utils.cache.yaml import load_yaml_any +from utils.roles.categories import categories_file from . import PROJECT_ROOT @@ -8,7 +9,7 @@ from . import PROJECT_ROOT class TestCategoryPaths(unittest.TestCase): @classmethod def setUpClass(cls): - file_path = str(PROJECT_ROOT / "roles" / "categories.yml") + file_path = str(categories_file()) data = load_yaml_any(file_path) cls.roles_def = data["roles"] diff --git a/tests/integration/roles/applications/id/test_basic.py b/tests/integration/roles/applications/id/test_basic.py index 54cde58d30..93a5644d04 100644 --- a/tests/integration/roles/applications/id/test_basic.py +++ b/tests/integration/roles/applications/id/test_basic.py @@ -3,6 +3,7 @@ from pathlib import Path from plugins.filter.invokable_paths import get_invokable_paths from utils.cache.yaml import load_yaml_any +from utils.roles.categories import categories_file from utils.roles.mapping import ROLE_FILE_VARS_MAIN @@ -19,9 +20,8 @@ class TestSysRolesApplicationId(unittest.TestCase): from . import PROJECT_ROOT cls.base_dir = str(PROJECT_ROOT) - cat_file = str(PROJECT_ROOT / "roles" / "categories.yml") + cat_file = str(categories_file()) cls.invokable_prefixes = set(get_invokable_paths(cat_file)) - # collect actual sys dirs roles_dir = Path(cls.base_dir) / "roles" cls.actual_dirs = [ str(p) @@ -35,7 +35,6 @@ class TestSysRolesApplicationId(unittest.TestCase): prefix = f"sys-{name.removeprefix('sys-')}" vars_file = str(Path(role_dir) / ROLE_FILE_VARS_MAIN) if prefix in self.invokable_prefixes: - # must exist with id self.assertTrue( Path(vars_file).is_file(), f"Missing vars/main.yml for invokable role {prefix}", @@ -47,7 +46,6 @@ class TestSysRolesApplicationId(unittest.TestCase): f"Expected 'application_id' in {vars_file} for invokable role {prefix}", ) else: - # if exists, must not contain id if not Path(vars_file).is_file(): continue content = load_yaml_any(vars_file) or {} diff --git a/tests/lint/ansible/roles/meta/test_main_galaxy_schema.py b/tests/lint/ansible/roles/meta/test_main_galaxy_schema.py index b0d5e29c14..5a3d0f830a 100644 --- a/tests/lint/ansible/roles/meta/test_main_galaxy_schema.py +++ b/tests/lint/ansible/roles/meta/test_main_galaxy_schema.py @@ -21,9 +21,9 @@ project-owned files (``meta/services.yml.``), not in the Galaxy slot. On top of the generic schema this lint also enforces two project-wide canonical exact-values to stop drift: ``galaxy_info.company`` is one specific block-scalar string (Kevin Veen-Birkenbach / -https://www.veen.world) and ``galaxy_info.platforms`` is one specific -5-distro list (ArchLinux/Debian/EL/Fedora/Ubuntu, each ``versions: -[all]``). Both fields had accumulated ~10 variants each (quoting style, +https://www.veen.world) and ``galaxy_info.platforms`` is the platform list +built from the ``meta/distros.yml`` SPOT (ArchLinux/Debian/EL/Fedora/Ubuntu, +each ``versions: [all]``). Both fields had accumulated ~10 variants each (quoting style, Unicode hyphen, typos, ``Linux``/``GenericLinux``/``Any`` platform names) before standardisation. The two-line company form keeps the parsed value under Galaxy's 50-character limit on ``company`` so galaxy-importer @@ -51,6 +51,7 @@ import yaml as _yaml from utils.cache.files import PROJECT_ROOT from utils.cache.yaml import load_yaml_any +from utils.distros import galaxy_platforms from utils.roles.mapping import ROLE_FILE_META_MAIN if TYPE_CHECKING: @@ -99,13 +100,7 @@ _REQUIRED_GALAXY_INFO: frozenset[str] = frozenset( _CANONICAL_COMPANY: str = "Kevin Veen-Birkenbach\nhttps://www.veen.world\n" -_CANONICAL_PLATFORMS: list[dict[str, Any]] = [ - {"name": "ArchLinux", "versions": ["all"]}, - {"name": "Debian", "versions": ["all"]}, - {"name": "EL", "versions": ["all"]}, - {"name": "Fedora", "versions": ["all"]}, - {"name": "Ubuntu", "versions": ["all"]}, -] +_CANONICAL_PLATFORMS: list[dict[str, Any]] = galaxy_platforms() def _meta_main_paths() -> list[Path]: diff --git a/tests/lint/ansible/roles/test_compose_resource_limits.py b/tests/lint/ansible/roles/test_compose_resource_limits.py index d5aed5281c..08a731bc70 100644 --- a/tests/lint/ansible/roles/test_compose_resource_limits.py +++ b/tests/lint/ansible/roles/test_compose_resource_limits.py @@ -21,7 +21,7 @@ declares the limits). Pure config toggles / cross-role integration flags that are not container-shaped are ignored. Scope: only roles whose directory name starts with an invokable prefix from -``roles/categories.yml`` are checked. Non-invokable categories (``dev-*``, +``meta/categories.yml`` are checked. Non-invokable categories (``dev-*``, ``sys-*``) ship no compose service of their own. Missing keys emit a ``::warning`` annotation each so CI annotates the source diff --git a/tests/lint/repository/test_distros_spot.py b/tests/lint/repository/test_distros_spot.py new file mode 100644 index 0000000000..c6ed40bd0d --- /dev/null +++ b/tests/lint/repository/test_distros_spot.py @@ -0,0 +1,154 @@ +"""Lint: ``meta/distros.yml`` is the only distro list. + +The Python layer reads the SPOT at runtime via :mod:`utils.distros`. Two +places cannot: ``default.env`` is parsed by ``scripts/meta/env/load.sh`` +before any Python exists, and a GitHub ``env:`` block is static YAML. Both +therefore carry a materialised copy, and these rules pin that copy to the +SPOT so a distro added in one place cannot go missing in the other. + +The image-reference rule guards the second half of the SPOT: the pkgmgr +and environment templates live in ``meta/distros.yml``; reintroducing an +interpolated ``pkgmgr-`` reference in code is the regression that +this file exists to catch. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +from utils.cache.files import read_text +from utils.cache.yaml import load_yaml_any +from utils.distros import ( + FILE_META_DISTROS, + IMAGE_PKGMGR, + IMAGE_PKGMGR_VIRGIN, + dev_runtime_images, + distro_names, + image_template, +) +from utils.env.parser import parse_static_env + +from . import PROJECT_ROOT + +WORKFLOWS_DIR = PROJECT_ROOT / ".github" / "workflows" +ENVIRONMENT_WORKFLOW = WORKFLOWS_DIR / "test-environment.yml" + +_INTERPOLATED_PKGMGR_RE = re.compile(r"pkgmgr-[{$]") +_SHELL_LOOP_SLUG = "${d}" +_LIST_SOURCES = ("*.yml", "*.sh", "*.md", "Makefile", "default.env") + + +class TestDistrosSpot(unittest.TestCase): + def setUp(self) -> None: + self.expected = " ".join(distro_names()) + + def test_default_env_declares_no_distro_list(self) -> None: + self.assertNotIn( + "INFINITO_DISTROS", + parse_static_env(PROJECT_ROOT / "default.env"), + "INFINITO_DISTROS is generated by utils/env/handlers/infinito/" + f"distros.py from {FILE_META_DISTROS}; a static default in " + "default.env would be a second source of truth.", + ) + + def test_every_literal_distro_list_matches_spot(self) -> None: + """A workflow ``env:``/``default:`` value, a shell array and a docs + table cannot call the SPOT, so they carry the list verbatim. Any run + of three or more declared distro names in a value position must + therefore spell the SPOT in full: a stale copy that lost a distro + fails here. Comment lines and YAML ``description:`` keys are prose + that names a subset as an example, so they are not value positions. + """ + run = re.compile( + r"(?:\b(?:{names})\b[\s,]+){{2,}}\b(?:{names})\b".format( + names="|".join(distro_names()) + ) + ) + this_file = Path(__file__).resolve() + offenders: list[str] = [] + for pattern in _LIST_SOURCES: + for path in sorted(PROJECT_ROOT.glob(f"**/{pattern}")): + if ".git" in path.parts or "node_modules" in path.parts: + continue + if path.resolve() == this_file: + continue + rel = path.relative_to(PROJECT_ROOT).as_posix() + for idx, line in enumerate(read_text(str(path)).splitlines(), 1): + stripped = line.strip() + if stripped.startswith(("#", "description:")): + continue + for found in run.findall(line): + listed = re.split(r"[\s,]+", found.strip()) + if listed != list(distro_names()): + offenders.append(f"{rel}:{idx}: {' '.join(listed)}") + + if offenders: + body = "\n".join(f" - {o}" for o in offenders) + self.fail( + f"Literal distro list(s) drifted from {FILE_META_DISTROS} " + f"(expected {self.expected!r}):\n{body}" + ) + + def test_environment_matrix_matches_spot(self) -> None: + parsed = load_yaml_any(str(ENVIRONMENT_WORKFLOW), default_if_missing={}) + matrix = parsed["jobs"]["test-environment"]["strategy"]["matrix"] + self.assertEqual( + tuple(matrix["dev_runtime_image"]), + dev_runtime_images(), + f"test-environment.yml dev_runtime_image matrix drifted from " + f"{FILE_META_DISTROS}.", + ) + + def _sources(self, pattern: str): + for path in sorted(PROJECT_ROOT.glob(f"**/{pattern}")): + if ".git" in path.parts or "node_modules" in path.parts: + continue + for idx, line in enumerate(read_text(str(path)).splitlines(), 1): + if _INTERPOLATED_PKGMGR_RE.search(line): + yield path.relative_to(PROJECT_ROOT).as_posix(), idx, line.strip() + + def test_python_and_make_compose_no_pkgmgr_reference(self) -> None: + offenders = [ + f"{rel}:{idx}: {line}" + for pattern in ("*.py", "Makefile") + for rel, idx, line in self._sources(pattern) + ] + if offenders: + body = "\n".join(f" - {o}" for o in offenders) + self.fail( + f"pkgmgr image reference composed outside {FILE_META_DISTROS}:\n" + f"{body}\n\nRender it via utils.distros.pkgmgr_image() or read " + "INFINITO_PARENT_IMAGE from the env layer." + ) + + def test_shell_pkgmgr_references_match_the_spot_template(self) -> None: + """Shell loops render ``{slug}`` from their own variable, so the SPOT + cannot be called per distro. The literal must still be the SPOT + template rendered with the declared owner and tag.""" + static = parse_static_env(PROJECT_ROOT / "default.env") + allowed = { + image_template(kind).format( + owner=static["INFINITO_PARENT_IMAGE_OWNER"], + slug=_SHELL_LOOP_SLUG, + tag=static["INFINITO_PARENT_IMAGE_TAG"], + ) + for kind in (IMAGE_PKGMGR, IMAGE_PKGMGR_VIRGIN) + } + + offenders = [ + f"{rel}:{idx}: {line}" + for rel, idx, line in self._sources("*.sh") + if not any(ref in line for ref in allowed) + ] + if offenders: + body = "\n".join(f" - {o}" for o in offenders) + self.fail( + f"Shell pkgmgr reference does not match a {FILE_META_DISTROS} " + f"template:\n{body}\n\nExpected one of: {sorted(allowed)}" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/cli/administration/inventory/provision/test_inventory_generator.py b/tests/unit/cli/administration/inventory/provision/test_inventory_generator.py index 8cda975519..373492521f 100644 --- a/tests/unit/cli/administration/inventory/provision/test_inventory_generator.py +++ b/tests/unit/cli/administration/inventory/provision/test_inventory_generator.py @@ -28,10 +28,7 @@ class TestInventoryGenerator(unittest.TestCase): ): data = generate_dynamic_inventory( host="localhost", - roles_dir=tmp / "roles", - categories_file=tmp / "categories.yml", tmp_inventory=tmp_inventory, - project_root=tmp / "repo", env={"PYTHONPATH": "x"}, ) diff --git a/tests/unit/cli/meta/categories/invokable/test_command.py b/tests/unit/cli/meta/categories/invokable/test_command.py index 513d0e88fc..f39765fe0c 100644 --- a/tests/unit/cli/meta/categories/invokable/test_command.py +++ b/tests/unit/cli/meta/categories/invokable/test_command.py @@ -31,7 +31,6 @@ class TestInvokableCategoriesCommand(unittest.TestCase): code = exc.code if exc.code is not None else 0 return int(code), out.getvalue(), err.getvalue() - # If no SystemExit was raised, it succeeded return 0, out.getvalue(), err.getvalue() @patch("cli.meta.categories.invokable.command.get_invokable_paths") @@ -46,7 +45,7 @@ class TestInvokableCategoriesCommand(unittest.TestCase): self.assertEqual(stderr, "") self.assertEqual(stdout.splitlines(), ["a/b", "c/d"]) - mock_invokable.assert_called_once_with(None, None) + mock_invokable.assert_called_once_with(suffix=None) mock_non_invokable.assert_not_called() @patch("cli.meta.categories.invokable.command.get_invokable_paths") @@ -62,20 +61,26 @@ class TestInvokableCategoriesCommand(unittest.TestCase): self.assertEqual(stderr, "") self.assertEqual(stdout.splitlines(), ["n1", "n2"]) - mock_non_invokable.assert_called_once_with(None, None) + mock_non_invokable.assert_called_once_with(suffix=None) mock_invokable.assert_not_called() @patch("cli.meta.categories.invokable.command.get_invokable_paths") - def test_roles_file_and_suffix_are_forwarded(self, mock_invokable) -> None: + def test_long_suffix_flag_is_forwarded(self, mock_invokable) -> None: mock_invokable.return_value = ["p1"] - code, stdout, stderr = self._run_main(["roles/categories.yml", "--suffix", "-"]) + code, stdout, stderr = self._run_main(["--suffix", "-"]) self.assertEqual(code, 0) self.assertEqual(stderr, "") self.assertEqual(stdout.splitlines(), ["p1"]) - mock_invokable.assert_called_once_with("roles/categories.yml", "-") + mock_invokable.assert_called_once_with(suffix="-") + + def test_positional_path_is_rejected(self) -> None: + code, _, stderr = self._run_main(["meta/categories.yml"]) + + self.assertEqual(code, 2) + self.assertIn("unrecognized arguments", stderr) @patch("cli.meta.categories.invokable.command.get_invokable_paths") def test_short_suffix_flag(self, mock_invokable) -> None: @@ -87,7 +92,7 @@ class TestInvokableCategoriesCommand(unittest.TestCase): self.assertEqual(stderr, "") self.assertEqual(stdout.splitlines(), ["p1"]) - mock_invokable.assert_called_once_with(None, "::") + mock_invokable.assert_called_once_with(suffix="::") @patch("cli.meta.categories.invokable.command.get_invokable_paths") def test_file_not_found_exits_1_and_prints_error(self, mock_invokable) -> None: diff --git a/tests/unit/plugins/filter/get/test_all_invokable_apps.py b/tests/unit/plugins/filter/get/test_all_invokable_apps.py index ba434db968..891cac8397 100644 --- a/tests/unit/plugins/filter/get/test_all_invokable_apps.py +++ b/tests/unit/plugins/filter/get/test_all_invokable_apps.py @@ -6,6 +6,7 @@ from unittest.mock import patch from plugins.filter.get.all.invokable_apps import get_all_invokable_apps from utils.cache.yaml import dump_yaml +from utils.roles.categories import categories_file from utils.roles.mapping import ROLE_FILE_VARS_MAIN @@ -15,7 +16,7 @@ class TestGetAllInvokableApps(unittest.TestCase): self.test_dir = Path(tempfile.mkdtemp(prefix="invokable_apps_test_")) self.roles_dir = self.test_dir / "roles" self.roles_dir.mkdir(parents=True, exist_ok=True) - self.categories_file = self.roles_dir / "categories.yml" + self.categories_file = categories_file(self.test_dir) categories = { "roles": { diff --git a/tests/unit/plugins/filter/get/test_entity_name.py b/tests/unit/plugins/filter/get/test_entity_name.py index 93e89ce797..6c1ad2f97c 100644 --- a/tests/unit/plugins/filter/get/test_entity_name.py +++ b/tests/unit/plugins/filter/get/test_entity_name.py @@ -1,11 +1,12 @@ -import os import shutil import sys import tempfile import unittest from pathlib import Path +from unittest.mock import patch from utils.cache.yaml import dump_yaml +from utils.roles.categories import categories_file class TestGetEntityNameFilter(unittest.TestCase): @@ -13,7 +14,7 @@ class TestGetEntityNameFilter(unittest.TestCase): self.temp_dir = tempfile.mkdtemp() self.roles_dir = str(Path(self.temp_dir) / "roles") Path(self.roles_dir).mkdir(parents=True) - self.categories_file = str(Path(self.roles_dir) / "categories.yml") + self.categories_file = str(categories_file(Path(self.temp_dir))) categories = { "roles": { @@ -35,10 +36,11 @@ class TestGetEntityNameFilter(unittest.TestCase): } dump_yaml(self.categories_file, categories) - self._cwd = str(Path.cwd()) - os.chdir(self.temp_dir) + root = patch("utils.roles.categories.PROJECT_ROOT", Path(self.temp_dir)) + root.start() + self.addCleanup(root.stop) - plugin_path = str(Path(self._cwd) / "plugins" / "filter") + plugin_path = str(Path.cwd() / "plugins" / "filter") if plugin_path not in sys.path and Path(plugin_path).is_dir(): sys.path.insert(0, plugin_path) from plugins.filter.get.entity_name import get_entity_name @@ -46,7 +48,6 @@ class TestGetEntityNameFilter(unittest.TestCase): self.get_entity_name = get_entity_name def tearDown(self): - os.chdir(self._cwd) shutil.rmtree(self.temp_dir) def test_entity_name_web_app(self): diff --git a/tests/unit/utils/cleanup/test_nginx_vhosts.py b/tests/unit/utils/cleanup/test_nginx_vhosts.py index 08c262be84..faac16c1b1 100644 --- a/tests/unit/utils/cleanup/test_nginx_vhosts.py +++ b/tests/unit/utils/cleanup/test_nginx_vhosts.py @@ -3,7 +3,6 @@ from __future__ import annotations import io -import os import shutil import tempfile import unittest @@ -18,6 +17,7 @@ from utils.cleanup.nginx_vhosts import ( main, purge_vhost_files_for_entities, ) +from utils.roles.categories import categories_file from utils.roles.mapping import ROLE_FILE_META_DOMAINS, ROLE_FILE_VARS_MAIN @@ -26,11 +26,10 @@ class NginxVhostsTestBase(unittest.TestCase): _reset_cache_for_tests() self.tmp = Path(tempfile.mkdtemp(prefix="nginx_vhosts_test_")) - # Roles tree (with categories.yml so `get_entity_name` resolves). self.roles_dir = self.tmp / "roles" self.roles_dir.mkdir(parents=True, exist_ok=True) dump_yaml( - self.roles_dir / "categories.yml", + categories_file(self.tmp), { "roles": { "web": { @@ -41,16 +40,14 @@ class NginxVhostsTestBase(unittest.TestCase): }, ) - # Fake nginx tree: /etc/nginx/conf.d/servers/{http,https}/ self.nginx_dir = self.tmp / "etc-nginx" self.servers_dir = self.nginx_dir / "conf.d" / "servers" (self.servers_dir / "http").mkdir(parents=True, exist_ok=True) (self.servers_dir / "https").mkdir(parents=True, exist_ok=True) - # categories.yml resolution leans on cwd → pin it to the tmp tree. - self._cwd = Path.cwd() - os.chdir(self.tmp) - self.addCleanup(lambda: os.chdir(self._cwd)) + root = patch("utils.roles.categories.PROJECT_ROOT", self.tmp) + root.start() + self.addCleanup(root.stop) self.addCleanup(lambda: shutil.rmtree(self.tmp, ignore_errors=True)) def _mk_role( @@ -87,7 +84,6 @@ class TestIterVhostFiles(NginxVhostsTestBase, unittest.TestCase): canonical=["matomo.infinito.example"], ) existing = self._touch_vhost("matomo.infinito.example", "https") - # http variant intentionally absent on disk got = list( iter_vhost_files_for_entity( @@ -211,7 +207,6 @@ class TestPurgeVhostFiles(NginxVhostsTestBase, unittest.TestCase): application_id="web-app-matomo", canonical=["matomo.infinito.example"], ) - # No vhost files placed on disk. removed = purge_vhost_files_for_entities( ["matomo"], @@ -282,8 +277,8 @@ class TestMainShim(NginxVhostsTestBase, unittest.TestCase): self.assertIn("usage:", stderr.getvalue()) def test_main_reports_noop_when_nothing_to_remove(self) -> None: + """With no roles and no vhost files, main still returns 0 and says so.""" stdout = io.StringIO() - # No roles, no vhost files — main MUST still return 0 with a no-op message. with patch.object(mod, "ROLES_DIR", self.roles_dir), redirect_stdout(stdout): rc = main(["nonexistent"]) self.assertEqual(rc, 0) diff --git a/tests/unit/utils/roles/test_entity_apps.py b/tests/unit/utils/roles/test_entity_apps.py index 0941bd4049..e16e97d5c2 100644 --- a/tests/unit/utils/roles/test_entity_apps.py +++ b/tests/unit/utils/roles/test_entity_apps.py @@ -2,13 +2,14 @@ from __future__ import annotations -import os import shutil import tempfile import unittest from pathlib import Path +from unittest.mock import patch from utils.cache.yaml import _reset_cache_for_tests, dump_yaml +from utils.roles.categories import categories_file from utils.roles.entity.apps import apps_for_entity from utils.roles.mapping import ROLE_FILE_VARS_MAIN @@ -21,7 +22,7 @@ class TestAppsForEntity(unittest.TestCase): self.roles_dir.mkdir(parents=True, exist_ok=True) dump_yaml( - self.roles_dir / "categories.yml", + categories_file(self.tmp), { "roles": { "web": { @@ -32,9 +33,9 @@ class TestAppsForEntity(unittest.TestCase): }, ) - self._cwd = Path.cwd() - os.chdir(self.tmp) - self.addCleanup(lambda: os.chdir(self._cwd)) + root = patch("utils.roles.categories.PROJECT_ROOT", self.tmp) + root.start() + self.addCleanup(root.stop) self.addCleanup(lambda: shutil.rmtree(self.tmp, ignore_errors=True)) def _mk_role(self, name: str, application_id: str | None) -> None: diff --git a/tests/unit/utils/roles/test_stage.py b/tests/unit/utils/roles/test_stage.py index cb95972a27..3b086c2ced 100644 --- a/tests/unit/utils/roles/test_stage.py +++ b/tests/unit/utils/roles/test_stage.py @@ -1,4 +1,4 @@ -"""Stage resolution reads roles/categories.yml as the SPOT: the ordered +"""Stage resolution reads meta/categories.yml as the SPOT: the ordered `stages` list, the per-category `stage`, and the run_after tiebreak that places roles in constructor -> workstation -> server -> destructor order.""" diff --git a/tests/unit/utils/roles/validation/test_invokable.py b/tests/unit/utils/roles/validation/test_invokable.py index ba8845e138..3518317c13 100644 --- a/tests/unit/utils/roles/validation/test_invokable.py +++ b/tests/unit/utils/roles/validation/test_invokable.py @@ -10,6 +10,7 @@ from unittest.mock import patch import utils.roles.validation.invokable as inv from utils.cache.yaml import dump_yaml +from utils.roles.categories import categories_file from utils.roles.mapping import ROLE_FILE_VARS_MAIN @@ -18,10 +19,8 @@ class TestInvokable(TestCase): self.tmp = Path(tempfile.mkdtemp(prefix="invokable_modutils_test_")) self.roles_dir = self.tmp / "roles" self.roles_dir.mkdir(parents=True, exist_ok=True) - self.categories_file = self.roles_dir / "categories.yml" + self.categories_file = categories_file(self.tmp) - # This structure triggers the YAML fallback in _get_invokable_paths() - # because it expects "categories", but we will patch _get_invokable_paths() anyway dump_yaml( self.categories_file, {"categories": [{"invokable_paths": ["web-app", "update", "util-desk"]}]}, @@ -71,7 +70,6 @@ class TestInvokable(TestCase): ): grouped = inv.list_invokables_by_type() - # server: web-app-* minus excluded oauth2 proxy (not present in test) self.assertIn("server", grouped) self.assertIn("workstation", grouped) self.assertIn("universal", grouped) @@ -79,14 +77,13 @@ class TestInvokable(TestCase): self.assertEqual(grouped["server"], sorted(["web-app-nextcloud", "matomo-app"])) self.assertEqual(grouped["workstation"], sorted(["util-desk-custom"])) - # universal = invokable - (server ∪ workstation) -> update remains self.assertEqual(grouped["universal"], ["update"]) def test_list_invokables_by_type_exclude(self) -> None: - # Exercise the exclude_re path of DeploymentTypeRule via a - # synthetic rule. The historical DEFAULT_RULES exclude (the now - # dissolved oauth2-proxy helper role) was removed when its - # sidecar logic was absorbed into web-app-keycloak. + """A synthetic rule exercises the exclude_re path of + DeploymentTypeRule. The historical DEFAULT_RULES exclude (the now + dissolved oauth2-proxy helper role) was removed when its sidecar + logic was absorbed into web-app-keycloak.""" rd = self.roles_dir / "web-app-excluded" (rd / "vars").mkdir(parents=True, exist_ok=True) dump_yaml(rd / ROLE_FILE_VARS_MAIN, {}) @@ -129,13 +126,14 @@ class TestInvokable(TestCase): excluded = inv.list_invokables_by_type(skip_mode="compose") kept = inv.list_invokables_by_type(skip_mode="swarm") - # web-app-matomo (app_id matomo-app) skips compose -> gone for compose, - # present for swarm; a role without a skip list stays in both. self.assertNotIn("matomo-app", excluded["server"]) self.assertIn("matomo-app", kept["server"]) self.assertIn("web-app-nextcloud", excluded["server"]) def test_types_from_group_names(self) -> None: + """``all`` and ``foo`` are not invokable and drop out; the three + invokable names map to server, workstation and the universal + leftover respectively.""" with ( patch.object(inv, "PROJECT_ROOT", self.tmp), patch.object( @@ -146,11 +144,11 @@ class TestInvokable(TestCase): ): got = inv.types_from_group_names( [ - "all", # not invokable -> ignored - "web-app-nextcloud", # invokable + matches server rule - "util-desk-custom", # invokable + matches workstation rule - "update", # invokable leftover -> universal - "foo", # not invokable -> ignored + "all", + "web-app-nextcloud", + "util-desk-custom", + "update", + "foo", ] ) diff --git a/tests/unit/utils/test_distros.py b/tests/unit/utils/test_distros.py new file mode 100644 index 0000000000..6a758477a2 --- /dev/null +++ b/tests/unit/utils/test_distros.py @@ -0,0 +1,70 @@ +"""The distro SPOT loader resolves names, families, images and platforms.""" + +from __future__ import annotations + +import unittest + +from utils.distros import ( + UnknownDistroError, + dev_runtime_image, + dev_runtime_images, + distro_family, + distro_names, + environment_image, + galaxy_platforms, + pkgmgr_image, +) + + +class TestDistroSpot(unittest.TestCase): + def test_names_are_the_build_matrix(self) -> None: + self.assertEqual( + distro_names(), ("arch", "debian", "ubuntu", "fedora", "centos") + ) + + def test_every_distro_maps_onto_an_os_family(self) -> None: + self.assertEqual( + distro_family(), + { + "arch": "Archlinux", + "debian": "Debian", + "ubuntu": "Debian", + "fedora": "RedHat", + "centos": "RedHat", + }, + ) + + def test_dev_runtime_images_follow_declaration_order(self) -> None: + self.assertEqual(dev_runtime_images()[0], dev_runtime_image("arch")) + self.assertEqual(len(dev_runtime_images()), len(distro_names())) + + def test_galaxy_platforms_are_sorted_by_name(self) -> None: + names = [entry["name"] for entry in galaxy_platforms()] + self.assertEqual(names, sorted(names)) + self.assertEqual(names, ["ArchLinux", "Debian", "EL", "Fedora", "Ubuntu"]) + self.assertTrue(all(e["versions"] == ["all"] for e in galaxy_platforms())) + + def test_galaxy_platforms_are_not_shared_between_calls(self) -> None: + first = galaxy_platforms() + first[0]["versions"].append("mutated") + self.assertEqual(galaxy_platforms()[0]["versions"], ["all"]) + + def test_pkgmgr_image_renders_the_template(self) -> None: + self.assertEqual( + pkgmgr_image("arch", owner="acme", tag="stable"), + "ghcr.io/acme/pkgmgr-arch:stable", + ) + + def test_environment_image_renders_the_template(self) -> None: + self.assertEqual( + environment_image("centos", owner="acme", repository="nexus", tag="ci-1"), + "ghcr.io/acme/nexus/centos:ci-1", + ) + + def test_unknown_distro_is_rejected(self) -> None: + with self.assertRaises(UnknownDistroError): + pkgmgr_image("gentoo", owner="acme", tag="stable") + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/distros.py b/utils/distros.py new file mode 100644 index 0000000000..38e292504a --- /dev/null +++ b/utils/distros.py @@ -0,0 +1,117 @@ +"""Distribution SPOT. + +``meta/distros.yml`` declares every distribution the project builds, tests +and deploys on: the id used as ``INFINITO_DISTRO``, the ``os_family`` +ansible reports for it, the Galaxy platform name, the upstream base image +the development-environment matrix boots it from, and the templates the +pkgmgr base image and the environment image references are rendered from. + +Consumers MUST read the distro set through this module. A local tuple of +distro names or an f-string spelling out ``ghcr.io/.../pkgmgr-...`` is a +second source of truth and drifts; ``tests/lint/repository/test_distros_spot.py`` +guards against reintroducing one. + +Registry owner, repository and tag are *not* declared here — they stay in +``default.env``, which is their SPOT. This module only renders them into +the templates. + +Keep this module free of Ansible imports and cheap to import: it is pulled +in by the env layer and by ``utils.packages.schema``, which reach contexts +where Ansible is not installed. +""" + +from __future__ import annotations + +from utils.cache import PROJECT_ROOT +from utils.cache.yaml import load_yaml + +FILE_META_DISTROS: str = "meta/distros.yml" + +IMAGE_PKGMGR = "pkgmgr" +IMAGE_PKGMGR_VIRGIN = "pkgmgr_virgin" +IMAGE_ENVIRONMENT = "environment" + + +class UnknownDistroError(ValueError): + """Raised for a distro id that ``meta/distros.yml`` does not declare.""" + + +def _doc() -> dict: + return load_yaml(str(PROJECT_ROOT / FILE_META_DISTROS)) + + +def _specs() -> dict[str, dict]: + return _doc()["distros"] + + +def _spec(distro: str) -> dict: + try: + return _specs()[distro] + except KeyError: + raise UnknownDistroError( + f"{distro!r} is not declared in {FILE_META_DISTROS}. " + f"Declared: {', '.join(distro_names())}." + ) from None + + +def distro_names() -> tuple[str, ...]: + """Every declared distro id, in declaration order.""" + return tuple(_specs()) + + +def distro_family() -> dict[str, str]: + """Map each distro id onto the ``os_family`` ansible reports for it.""" + return {name: spec["family"] for name, spec in _specs().items()} + + +def galaxy_platforms() -> list[dict[str, object]]: + """Canonical ``galaxy_info.platforms`` block for every role's meta/main.yml. + + Sorted by platform name, which is the order the role files carry and + which the schema lint compares against verbatim. + """ + versions = list(_doc()["galaxy_platform_versions"]) + return [ + {"name": name, "versions": list(versions)} + for name in sorted(spec["galaxy_platform"] for spec in _specs().values()) + ] + + +def dev_runtime_images() -> tuple[str, ...]: + """Upstream base images of the development-environment test matrix.""" + return tuple(spec["dev_runtime_image"] for spec in _specs().values()) + + +def dev_runtime_image(distro: str) -> str: + """Upstream base image the development environment boots a distro from.""" + return _spec(distro)["dev_runtime_image"] + + +def image_template(kind: str) -> str: + """Raw reference template, for callers rendering a non-distro slug. + + The shell-side installer scripts loop over distros with their own + variable, so they render ``{slug}`` themselves rather than per distro. + """ + return _doc()["images"][kind] + + +def _render(kind: str, distro: str, **fields: str) -> str: + return image_template(kind).format(slug=_spec(distro)["slug"], **fields) + + +def pkgmgr_image(distro: str, owner: str, tag: str) -> str: + """Reference of the pkgmgr base image a distro's container builds FROM.""" + return _render(IMAGE_PKGMGR, distro, owner=owner, tag=tag) + + +def pkgmgr_virgin_image(distro: str, owner: str, tag: str) -> str: + """Reference of the untouched pkgmgr image the install tests boot.""" + return _render(IMAGE_PKGMGR_VIRGIN, distro, owner=owner, tag=tag) + + +def environment_image(distro: str, owner: str, repository: str, tag: str) -> str: + """Reference of the environment image built for a distro.""" + return _render( + IMAGE_ENVIRONMENT, distro, owner=owner, repository=repository, tag=tag + ) diff --git a/utils/env/handlers/__init__.py b/utils/env/handlers/__init__.py index a82154c8b1..bfaffd8d14 100644 --- a/utils/env/handlers/__init__.py +++ b/utils/env/handlers/__init__.py @@ -17,6 +17,7 @@ from . import ( from .infinito import ( ca_cert_host, container, + distros, docker_volume, image, image_repository, @@ -49,6 +50,7 @@ from .infinito.worker import fetch as worker_fetch ORDERED_HANDLERS = [ passthrough, + distros, dir_var_lib, dir_backups, dir_secrets, diff --git a/utils/env/handlers/infinito/distros.py b/utils/env/handlers/infinito/distros.py new file mode 100644 index 0000000000..3e2571288c --- /dev/null +++ b/utils/env/handlers/infinito/distros.py @@ -0,0 +1,21 @@ +"""INFINITO_DISTROS: every distro id declared in meta/distros.yml. + +The key carries no static default; it is generated here so the shell and +CI side of the project read the same SPOT the Python side does. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from utils.distros import FILE_META_DISTROS, distro_names + +if TYPE_CHECKING: + from utils.env.builder import BuildContext, EnvBuilder + +KEY = "INFINITO_DISTROS" +COMMENT = f"Space-separated distro ids, derived from {FILE_META_DISTROS}." + + +def apply(eb: EnvBuilder, ctx: BuildContext) -> None: + eb.set(KEY, " ".join(distro_names()), comment=COMMENT) diff --git a/utils/env/handlers/infinito/image.py b/utils/env/handlers/infinito/image.py index 966241f69e..810b4eab0d 100644 --- a/utils/env/handlers/infinito/image.py +++ b/utils/env/handlers/infinito/image.py @@ -5,6 +5,8 @@ from __future__ import annotations from typing import TYPE_CHECKING +from utils.distros import environment_image + if TYPE_CHECKING: from utils.env.builder import BuildContext, EnvBuilder @@ -23,6 +25,6 @@ def apply(eb: EnvBuilder, ctx: BuildContext) -> None: tag = eb.get("INFINITO_IMAGE_TAG") eb.setdefault( KEY, - f"ghcr.io/{owner}/{repo_name}/{distro}:{tag}", + environment_image(distro, owner=owner, repository=repo_name, tag=tag), comment=COMMENT, ) diff --git a/utils/env/handlers/infinito/parent_image.py b/utils/env/handlers/infinito/parent_image.py index 8dd8d69843..1a244dbc75 100644 --- a/utils/env/handlers/infinito/parent_image.py +++ b/utils/env/handlers/infinito/parent_image.py @@ -5,6 +5,8 @@ from __future__ import annotations from typing import TYPE_CHECKING +from utils.distros import pkgmgr_image + if TYPE_CHECKING: from utils.env.builder import BuildContext, EnvBuilder @@ -16,7 +18,7 @@ COMMENT = ( def compose(owner: str, distro: str, tag: str) -> str: - return f"ghcr.io/{owner}/pkgmgr-{distro}:{tag}" + return pkgmgr_image(distro, owner=owner, tag=tag) def apply(eb: EnvBuilder, ctx: BuildContext) -> None: diff --git a/utils/env/handlers/passthrough.py b/utils/env/handlers/passthrough.py index 1ac5b107cf..18b3507658 100644 --- a/utils/env/handlers/passthrough.py +++ b/utils/env/handlers/passthrough.py @@ -67,7 +67,6 @@ STATIC_KEYS = ( "INFINITO_TEST_PATTERN", "INFINITO_TEST_RUNNER", "INFINITO_LINT_RUNNER", - "INFINITO_DISTROS", "INFINITO_APP_DISCOVERY_RUNNER", "INFINITO_DEPLOY_MODE", "INFINITO_DISCOVERY_SORT", diff --git a/utils/packages/schema.py b/utils/packages/schema.py index 7b5be09802..1edc78891e 100644 --- a/utils/packages/schema.py +++ b/utils/packages/schema.py @@ -51,6 +51,8 @@ from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING, Any +from utils.distros import distro_family + if TYPE_CHECKING: from pathlib import Path @@ -89,15 +91,10 @@ REPO_KEYS: frozenset[str] = frozenset( """A repository definition must declare at least one of these; the rest (``name``, ``description``, ``gpgkey``, ``gpgcheck``, ``state``) qualify it.""" -DISTRO_FAMILY: dict[str, str] = { - "arch": "Archlinux", - "debian": "Debian", - "ubuntu": "Debian", - "fedora": "RedHat", - "centos": "RedHat", -} +DISTRO_FAMILY: dict[str, str] = distro_family() """The default distro matrix mapped onto the ``os_family`` ansible reports -for it. Coverage is required for every key.""" +for it, read from the ``meta/distros.yml`` SPOT. Coverage is required for +every key.""" class PackagesShapeError(ValueError): diff --git a/utils/roles/categories.py b/utils/roles/categories.py new file mode 100644 index 0000000000..f8fc7ffa6e --- /dev/null +++ b/utils/roles/categories.py @@ -0,0 +1,75 @@ +"""Category-tree SPOT. + +``meta/categories.yml`` at the repository root declares the nested +category tree every role name is derived from: the dash-joined path of a +node is the role's category prefix (``web`` -> ``app`` yields +``web-app-*``), and nodes carry the ``invokable``, ``stage``, ``modes`` +and ``run_after`` policy the dispatcher, the stage lookup and the +inventory generator read. + +This module owns the file's location and the tree-walking primitives. +Consumers MUST resolve the path through :func:`categories_file` rather +than spelling it themselves, so a future move touches one line. + +:data:`FILE_META_CATEGORIES` stays repository-root *relative* and is not +pre-joined onto ``PROJECT_ROOT`` at import time on purpose: the root is +not fixed. ``utils.roles.validation.invokable`` is exercised against a +patched ``PROJECT_ROOT`` and test fixtures build the file inside a +temporary tree. A module level absolute constant would freeze the root at +import time and silently ignore both. + +The file is not optional. It lives beside ``roles/``, which the packaging +config excludes from the wheel just as it excludes ``meta/``, so any +context that can resolve a role name also carries this file. A missing +file is a broken checkout and raises rather than degrading to an empty +category tree, which would silently strip every category prefix off every +role name. + +Keep this module free of Ansible imports and cheap to import: it is +pulled in by ``plugins/filter/invokable_paths.py``, which runs in +contexts where Ansible is not installed (see +``tests/unit/utils/cache/test_data.py``). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from utils.cache import PROJECT_ROOT +from utils.cache.yaml import load_yaml + +if TYPE_CHECKING: + from pathlib import Path + +FILE_META_CATEGORIES: str = "meta/categories.yml" + + +def categories_file(root: Path | None = None) -> Path: + """Absolute path of the category file below *root*. + + Args: + root: tree to resolve against; the repository root when omitted. + """ + return (PROJECT_ROOT if root is None else root) / FILE_META_CATEGORIES + + +def load_categories_tree(categories_file): + """Return the ``roles`` mapping of the category file. + + Args: + categories_file: path-like of a ``meta/categories.yml``. + """ + return load_yaml(categories_file)["roles"] + + +def flatten_categories(tree, prefix=""): + """Flattens nested category tree to all possible category paths.""" + result = [] + for k, v in tree.items(): + current = f"{prefix}-{k}" if prefix else k + result.append(current) + if isinstance(v, dict): + for sk, sv in v.items(): + if isinstance(sv, dict): + result.extend(flatten_categories({sk: sv}, current)) + return result diff --git a/utils/roles/entity/name.py b/utils/roles/entity/name.py index dc252d6ac9..52ca05517f 100644 --- a/utils/roles/entity/name.py +++ b/utils/roles/entity/name.py @@ -1,24 +1,8 @@ -from pathlib import Path - -from utils import PROJECT_ROOT -from utils.cache.yaml import load_yaml - - -def load_categories_tree(categories_file): - return load_yaml(categories_file)["roles"] - - -def flatten_categories(tree, prefix=""): - """Flattens nested category tree to all possible category paths.""" - result = [] - for k, v in tree.items(): - current = f"{prefix}-{k}" if prefix else k - result.append(current) - if isinstance(v, dict): - for sk, sv in v.items(): - if isinstance(sv, dict): - result.extend(flatten_categories({sk: sv}, current)) - return result +from utils.roles.categories import ( + categories_file, + flatten_categories, + load_categories_tree, +) def get_entity_name(role_name): @@ -26,20 +10,7 @@ def get_entity_name(role_name): Get the entity name from a role name by removing the longest matching category path from categories.yml. """ - possible_locations = [ - str(Path(str(Path.cwd())) / "roles" / "categories.yml"), - str(PROJECT_ROOT / "roles" / "categories.yml"), - "roles/categories.yml", - ] - categories_file = None - for loc in possible_locations: - if Path(loc).exists(): - categories_file = loc - break - if not categories_file: - return role_name - - categories_tree = load_categories_tree(categories_file) + categories_tree = load_categories_tree(str(categories_file())) all_category_paths = flatten_categories(categories_tree) role_name_lc = role_name.lower() diff --git a/utils/roles/required_by_coverage.py b/utils/roles/required_by_coverage.py index a92057392a..1d342dfcc3 100644 --- a/utils/roles/required_by_coverage.py +++ b/utils/roles/required_by_coverage.py @@ -10,6 +10,7 @@ from pathlib import Path from utils.cache.files import read_text from utils.cache.yaml import load_yaml_any +from utils.roles.categories import categories_file from utils.roles.mapping import ROLE_FILE_META_SERVICES from . import PROJECT_ROOT @@ -23,8 +24,8 @@ def role_is_invokable(role_id: str, roles_dir: str | Path | None = None) -> bool """True when any node along `role_id`'s category path has `invokable: true`.""" if not role_id: return False - base = _default_roles_dir(roles_dir) - tree_doc = load_yaml_any(str(base / "categories.yml"), default_if_missing={}) + spot = categories_file(_default_roles_dir(roles_dir).parent) + tree_doc = load_yaml_any(str(spot), default_if_missing={}) tree = tree_doc.get("roles") or {} if isinstance(tree_doc, dict) else {} node = tree diff --git a/utils/roles/stage.py b/utils/roles/stage.py index db4a46dc59..08b84acff6 100644 --- a/utils/roles/stage.py +++ b/utils/roles/stage.py @@ -1,6 +1,6 @@ """Deploy-stage lookup SPOT. -``roles/categories.yml`` declares the ordered ``stages`` (constructor -> +``meta/categories.yml`` declares the ordered ``stages`` (constructor -> workstation -> server -> destructor, matching ``tasks/stages/0*.yml``) and a ``stage`` per top-level category. This module resolves a role to its stage and exposes the canonical stage order, so both the CLI (``cli.meta.roles.order``) @@ -9,29 +9,16 @@ and the ``stage_order`` / ``role_stage`` lookups read one source of truth. from __future__ import annotations -from pathlib import Path - from utils import PROJECT_ROOT from utils.cache.yaml import load_yaml from utils.roles.applications.services.registry import run_after_topological_order +from utils.roles.categories import categories_file DEFAULT_STAGE = "server" -def _categories_file() -> str | None: - for loc in ( - Path.cwd() / "roles" / "categories.yml", - PROJECT_ROOT / "roles" / "categories.yml", - Path("roles/categories.yml"), - ): - if Path(loc).exists(): - return str(loc) - return None - - def _doc() -> dict: - f = _categories_file() - return load_yaml(f) if f else {} + return load_yaml(str(categories_file())) _META_DICT_KEYS = frozenset({"modes"}) diff --git a/utils/roles/validation/invokable.py b/utils/roles/validation/invokable.py index b04f812fcc..c89378d804 100644 --- a/utils/roles/validation/invokable.py +++ b/utils/roles/validation/invokable.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING from utils.cache.yaml import load_yaml_any +from utils.roles.categories import categories_file from utils.roles.mapping import ROLE_FILE_VARS_MAIN from . import PROJECT_ROOT @@ -32,7 +33,6 @@ DEFAULT_RULES: tuple[DeploymentTypeRule, ...] = ( include_re=re.compile(r"^(desk-|util-desk-)"), exclude_re=None, ), - # "universal": everything invokable that is not matched by server/workstation rules DeploymentTypeRule( name="universal", include_re=re.compile(r".*"), @@ -45,10 +45,6 @@ def _roles_dir() -> Path: return PROJECT_ROOT / "roles" -def _categories_file() -> Path: - return _roles_dir() / "categories.yml" - - def _read_yaml(path: Path) -> dict: if not path.is_file(): return {} @@ -57,10 +53,12 @@ def _read_yaml(path: Path) -> dict: def _role_lifecycle(role_dir: Path) -> str: - # `lifecycle` lives on the role's primary entity at - # `meta/services.yml..lifecycle`. Delegate to the - # canonical helper and treat any failure as "no lifecycle" so role - # discovery is never broken by a single malformed meta file. + """Lifecycle of the role's primary entity, empty when unresolvable. + + Reads ``meta/services.yml..lifecycle`` through the + canonical helper. Any failure degrades to "no lifecycle" so a single + malformed meta file cannot break role discovery. + """ from utils.roles.meta_lookup import get_role_lifecycle try: @@ -71,8 +69,9 @@ def _role_lifecycle(role_dir: Path) -> str: def _role_skip_modes(role_dir: Path) -> list[str]: - # `skip` lists deploy modes (compose/swarm) the role opts out of in - # test-deploy discovery; never break discovery on a malformed meta file. + """Deploy modes (compose/swarm) the role opts out of in test-deploy + discovery. A malformed meta file degrades to "skips nothing" rather + than breaking discovery.""" from utils.roles.meta_lookup import get_role_skip try: @@ -84,15 +83,14 @@ def _role_skip_modes(role_dir: Path) -> list[str]: def _get_invokable_paths() -> list[str]: from plugins.filter.invokable_paths import get_invokable_paths - paths = get_invokable_paths(str(_categories_file())) + paths = get_invokable_paths(str(categories_file(PROJECT_ROOT))) if not paths: raise RuntimeError("No invokable paths found in categories.yml") return [str(p) for p in paths] def _is_role_invokable(role_name: str, invokable_paths: Iterable[str]) -> bool: - # Matches your existing logic: - # role == p or role.startswith(p + "-") + """True when the role is one of the invokable paths or sits below one.""" return any(role_name == p or role_name.startswith(p + "-") for p in invokable_paths) @@ -157,7 +155,6 @@ def list_invokables_by_type( if not invokable_paths or not roles_dir.is_dir(): return {r.name: [] for r in rules} - # Gather invokable role dirs first (+ optional lifecycle gating) invokable_role_dirs: list[Path] = [] for role_dir in sorted( [p for p in roles_dir.iterdir() if p.is_dir()], key=lambda p: p.name @@ -175,13 +172,11 @@ def list_invokables_by_type( invokable_role_dirs.append(role_dir) - # Identify non-universal rules for subtraction logic rules_list = list(rules) non_universal = [r for r in rules_list if r.name != "universal"] by_type: dict[str, list[str]] = {r.name: [] for r in rules_list} - # First pass: server/workstation buckets claimed_role_names: set[str] = set() for role_dir in invokable_role_dirs: for r in non_universal: @@ -190,13 +185,11 @@ def list_invokables_by_type( claimed_role_names.add(role_dir.name) break - # Second pass: universal = remaining invokables if "universal" in by_type: for role_dir in invokable_role_dirs: if role_dir.name not in claimed_role_names: by_type["universal"].append(_role_to_app_id(role_dir)) - # Normalize sort + unique for k, v in by_type.items(): by_type[k] = sorted(set(v)) @@ -231,7 +224,6 @@ def types_from_group_names( matched: set[str] = set() claimed: set[str] = set() - # server/workstation via rules for g in invokable_names: for r in non_universal: if _rule_matches_role_name(r, g): @@ -239,7 +231,6 @@ def types_from_group_names( claimed.add(g) break - # universal = invokable leftovers (not claimed by server/workstation) if any(g not in claimed for g in invokable_names): matched.add("universal")