mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-08 11:46:35 +00:00
New lint guard
==============
tests/lint/repository/test_no_direct_yaml_calls.py AST-walks the
production tree (utils/, cli/, plugins/, filter_plugins/,
lookup_plugins/, roles/, scripts/, tasks/) and flags any direct
`yaml.safe_load` / `yaml.safe_dump` / `yaml.load` / `yaml.dump` /
`yaml.safe_load_all` / `yaml.dump_all` call. Aliased imports are
tracked: `import yaml as Y` ⇒ `Y.safe_load(...)` is flagged;
`from yaml import safe_load` ⇒ bare `safe_load(...)` is flagged.
Per-line opt-out: `# noqa: direct-yaml` on the call's physical line
(or any line spanned by a multi-line call). No central exemption
list — every legitimate exception lives next to its call with a
human-readable rationale in the surrounding comment.
Cache module additions
======================
* `dump_yaml_str(data)` — symmetric to `safe_dump(data)` with no file
path. Used for stdout output and filter return values where the
cache cannot help but the API needs to stay consistent.
* `load_yaml_str(text)` — symmetric to `safe_load(text)`. Used when
YAML comes from an HTTP body, a mutated buffer, or a mock-patched
source whose content does not correspond to a stable on-disk file.
Migrated production call sites (33 → 0 direct-yaml violations)
==============================================================
* utils/{entity_name_utils, valid_deploy_id, invokable, service_registry,
domains/list, applications/{config, in_group_deps},
docker/image/discovery, meta/{port_bands, scan},
roles/{dependency_resolver, meta_lookup, resource_validation}}
* cli/{build/{graph, inventory/full/command}, create/inventory/{host_vars,
mirror_overrides}, fix/vars_main_files, meta/{applications/in_group_deps,
callorder}, mirror/resolver, validate/inventory}
* plugins/{filter/{compose_volumes, get_all_application_ids, get_role,
invokable_paths, role_path_by_app_id}, lookup/applications_current_play}
* roles/{svc-db-postgres/filter_plugins/split_postgres_connections,
web-app-bigbluebutton/filter_plugins/compose_mods,
web-app-dashboard/lookup_plugins/docker_cards}
Behaviour adjustments
=====================
* utils/roles/dependency_resolver — `safe_load_all` (multi-doc) →
single-doc `load_yaml_any` wrapped in a list. Ansible task files are
single-doc by convention; the downstream multi-doc loop stays intact
for forward compatibility.
* roles/sys-svc-compose-ca/files/compose_ca.py — KEEPS direct
yaml.safe_load / yaml.safe_dump (annotated with noqa). The script is
copied to the target host via `ansible.builtin.copy` and the target
has no `utils/` package on PYTHONPATH.
noqa-annotated direct-yaml call sites (with rationale)
======================================================
* utils/cache/yaml.py — IS the cache; would recurse.
* utils/handler/yaml.py — vault-aware Loader subclass that wraps
\!vault scalars in VaultScalar; cache-by-path collapses the wrapping.
* cli/encrypt/inventory/__main__.py — vault-aware Dumper subclass that
emits \!vault literal blocks; the cached dump_yaml writes plain YAML
which is the wrong output here.
* roles/sys-svc-compose-ca/files/compose_ca.py — runtime-deployed.
* tasks/utils/migrate_meta_layout.py — once-off migration script.
Validation
==========
* Unit tests: 1573 passed.
* Lint: 56 passed (incl. new test_no_direct_yaml_calls).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
235 lines
6.9 KiB
Python
235 lines
6.9 KiB
Python
# utils/invokable.py
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Iterable, Set
|
|
|
|
from utils.cache.yaml import load_yaml_any
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DeploymentTypeRule:
|
|
name: str
|
|
include_re: re.Pattern[str]
|
|
exclude_re: re.Pattern[str] | None
|
|
|
|
|
|
DEFAULT_RULES: tuple[DeploymentTypeRule, ...] = (
|
|
DeploymentTypeRule(
|
|
name="server",
|
|
include_re=re.compile(r"^(web-app-|web-svc-)"),
|
|
exclude_re=re.compile(r"^(web-app-oauth2-proxy)$"),
|
|
),
|
|
DeploymentTypeRule(
|
|
name="workstation",
|
|
include_re=re.compile(r"^(desk-|util-desk-)"),
|
|
exclude_re=None,
|
|
),
|
|
# "universal": everything invokable that is not matched by server/workstation rules
|
|
DeploymentTypeRule(
|
|
name="universal",
|
|
include_re=re.compile(r".*"),
|
|
exclude_re=None,
|
|
),
|
|
)
|
|
|
|
|
|
def _repo_root() -> Path:
|
|
# utils/invokable.py -> utils -> repo root
|
|
return Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def _roles_dir() -> Path:
|
|
return _repo_root() / "roles"
|
|
|
|
|
|
def _categories_file() -> Path:
|
|
return _roles_dir() / "categories.yml"
|
|
|
|
|
|
def _read_yaml(path: Path) -> dict:
|
|
if not path.is_file():
|
|
return {}
|
|
data = load_yaml_any(str(path), default_if_missing={})
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
|
|
def _role_lifecycle(role_dir: Path) -> str:
|
|
# Per req-010 `lifecycle` lives on the role's primary entity at
|
|
# `meta/services.yml.<primary_entity>.lifecycle`. Delegate to the
|
|
# canonical helper and treat any failure as "no lifecycle" so role
|
|
# discovery is never broken by a single malformed meta file.
|
|
from utils.roles.meta_lookup import get_role_lifecycle
|
|
|
|
try:
|
|
value = get_role_lifecycle(role_dir, role_name=role_dir.name)
|
|
except Exception:
|
|
return ""
|
|
return value or ""
|
|
|
|
|
|
def _get_invokable_paths() -> list[str]:
|
|
from plugins.filter.invokable_paths import get_invokable_paths # type: ignore
|
|
|
|
paths = get_invokable_paths(str(_categories_file()))
|
|
if not paths:
|
|
raise RuntimeError("No invokable paths found in categories.yml")
|
|
return [str(p) for p in paths]
|
|
|
|
|
|
def _is_role_invokable(role_name: str, invokable_paths: Iterable[str]) -> bool:
|
|
# Matches your existing logic:
|
|
# role == p or role.startswith(p + "-")
|
|
for p in invokable_paths:
|
|
if role_name == p or role_name.startswith(p + "-"):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _role_to_app_id(role_dir: Path) -> str:
|
|
vars_file = role_dir / "vars" / "main.yml"
|
|
if not vars_file.is_file():
|
|
return role_dir.name
|
|
|
|
try:
|
|
data = _read_yaml(vars_file)
|
|
app_id = data.get("application_id")
|
|
return str(app_id) if app_id else role_dir.name
|
|
except Exception:
|
|
return role_dir.name
|
|
|
|
|
|
def list_invokable_app_ids() -> list[str]:
|
|
roles_dir = _roles_dir()
|
|
invokable_paths = _get_invokable_paths()
|
|
if not invokable_paths or not roles_dir.is_dir():
|
|
return []
|
|
|
|
result: list[str] = []
|
|
for role_dir in sorted(
|
|
[p for p in roles_dir.iterdir() if p.is_dir()], key=lambda p: p.name
|
|
):
|
|
if _is_role_invokable(role_dir.name, invokable_paths):
|
|
result.append(_role_to_app_id(role_dir))
|
|
|
|
return sorted(set(result))
|
|
|
|
|
|
def _rule_matches_role_name(rule: DeploymentTypeRule, role_name: str) -> bool:
|
|
if not rule.include_re.search(role_name):
|
|
return False
|
|
if rule.exclude_re and rule.exclude_re.search(role_name):
|
|
return False
|
|
return True
|
|
|
|
|
|
def list_invokables_by_type(
|
|
*,
|
|
rules: Iterable[DeploymentTypeRule] = DEFAULT_RULES,
|
|
lifecycles: Set[str] | None = None,
|
|
) -> dict[str, list[str]]:
|
|
"""
|
|
Returns:
|
|
{
|
|
"server": [...],
|
|
"workstation": [...],
|
|
"universal": [...],
|
|
}
|
|
|
|
"universal" = invokable roles that are NOT matched by any other non-universal rule.
|
|
|
|
If lifecycles is provided, only roles whose galaxy_info.lifecycle is in the set
|
|
are included. Missing lifecycle is treated as "not matching".
|
|
"""
|
|
roles_dir = _roles_dir()
|
|
invokable_paths = _get_invokable_paths()
|
|
if not invokable_paths or not roles_dir.is_dir():
|
|
return {r.name: [] for r in rules}
|
|
|
|
# Gather invokable role dirs first (+ optional lifecycle gating)
|
|
invokable_role_dirs: list[Path] = []
|
|
for role_dir in sorted(
|
|
[p for p in roles_dir.iterdir() if p.is_dir()], key=lambda p: p.name
|
|
):
|
|
if not _is_role_invokable(role_dir.name, invokable_paths):
|
|
continue
|
|
|
|
if lifecycles is not None:
|
|
lc = _role_lifecycle(role_dir)
|
|
if not lc or lc not in lifecycles:
|
|
continue
|
|
|
|
invokable_role_dirs.append(role_dir)
|
|
|
|
# Identify non-universal rules for subtraction logic
|
|
rules_list = list(rules)
|
|
non_universal = [r for r in rules_list if r.name != "universal"]
|
|
|
|
by_type: dict[str, list[str]] = {r.name: [] for r in rules_list}
|
|
|
|
# First pass: server/workstation buckets
|
|
claimed_role_names: set[str] = set()
|
|
for role_dir in invokable_role_dirs:
|
|
for r in non_universal:
|
|
if _rule_matches_role_name(r, role_dir.name):
|
|
by_type[r.name].append(_role_to_app_id(role_dir))
|
|
claimed_role_names.add(role_dir.name)
|
|
break
|
|
|
|
# Second pass: universal = remaining invokables
|
|
if "universal" in by_type:
|
|
for role_dir in invokable_role_dirs:
|
|
if role_dir.name not in claimed_role_names:
|
|
by_type["universal"].append(_role_to_app_id(role_dir))
|
|
|
|
# Normalize sort + unique
|
|
for k, v in by_type.items():
|
|
by_type[k] = sorted(set(v))
|
|
|
|
return by_type
|
|
|
|
|
|
def types_from_group_names(
|
|
group_names: Iterable[str],
|
|
*,
|
|
rules: Iterable[DeploymentTypeRule] = DEFAULT_RULES,
|
|
) -> list[str]:
|
|
"""
|
|
SPOT:
|
|
- invokable is defined by categories.yml via _get_invokable_paths()/_is_role_invokable()
|
|
- server/workstation/universal classification is defined by DEFAULT_RULES
|
|
|
|
Semantics:
|
|
universal = invokable AND NOT matched by any non-universal rule.
|
|
"""
|
|
names = [str(g).strip() for g in (group_names or []) if str(g).strip()]
|
|
if not names:
|
|
return []
|
|
|
|
invokable_paths = _get_invokable_paths()
|
|
invokable_names = [g for g in names if _is_role_invokable(g, invokable_paths)]
|
|
if not invokable_names:
|
|
return []
|
|
|
|
rules_list = list(rules)
|
|
non_universal = [r for r in rules_list if r.name != "universal"]
|
|
|
|
matched: set[str] = set()
|
|
claimed: set[str] = set()
|
|
|
|
# server/workstation via rules
|
|
for g in invokable_names:
|
|
for r in non_universal:
|
|
if _rule_matches_role_name(r, g):
|
|
matched.add(r.name)
|
|
claimed.add(g)
|
|
break
|
|
|
|
# universal = invokable leftovers (not claimed by server/workstation)
|
|
if any(g not in claimed for g in invokable_names):
|
|
matched.add("universal")
|
|
|
|
return sorted(matched)
|