mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-13 14:16:46 +00:00
New lint guard
==============
tests/lint/repository/test_no_direct_yaml_calls.py AST-walks the
production tree (utils/, cli/, plugins/, filter_plugins/,
lookup_plugins/, roles/, scripts/, tasks/) and flags any direct
`yaml.safe_load` / `yaml.safe_dump` / `yaml.load` / `yaml.dump` /
`yaml.safe_load_all` / `yaml.dump_all` call. Aliased imports are
tracked: `import yaml as Y` ⇒ `Y.safe_load(...)` is flagged;
`from yaml import safe_load` ⇒ bare `safe_load(...)` is flagged.
Per-line opt-out: `# noqa: direct-yaml` on the call's physical line
(or any line spanned by a multi-line call). No central exemption
list — every legitimate exception lives next to its call with a
human-readable rationale in the surrounding comment.
Cache module additions
======================
* `dump_yaml_str(data)` — symmetric to `safe_dump(data)` with no file
path. Used for stdout output and filter return values where the
cache cannot help but the API needs to stay consistent.
* `load_yaml_str(text)` — symmetric to `safe_load(text)`. Used when
YAML comes from an HTTP body, a mutated buffer, or a mock-patched
source whose content does not correspond to a stable on-disk file.
Migrated production call sites (33 → 0 direct-yaml violations)
==============================================================
* utils/{entity_name_utils, valid_deploy_id, invokable, service_registry,
domains/list, applications/{config, in_group_deps},
docker/image/discovery, meta/{port_bands, scan},
roles/{dependency_resolver, meta_lookup, resource_validation}}
* cli/{build/{graph, inventory/full/command}, create/inventory/{host_vars,
mirror_overrides}, fix/vars_main_files, meta/{applications/in_group_deps,
callorder}, mirror/resolver, validate/inventory}
* plugins/{filter/{compose_volumes, get_all_application_ids, get_role,
invokable_paths, role_path_by_app_id}, lookup/applications_current_play}
* roles/{svc-db-postgres/filter_plugins/split_postgres_connections,
web-app-bigbluebutton/filter_plugins/compose_mods,
web-app-dashboard/lookup_plugins/docker_cards}
Behaviour adjustments
=====================
* utils/roles/dependency_resolver — `safe_load_all` (multi-doc) →
single-doc `load_yaml_any` wrapped in a list. Ansible task files are
single-doc by convention; the downstream multi-doc loop stays intact
for forward compatibility.
* roles/sys-svc-compose-ca/files/compose_ca.py — KEEPS direct
yaml.safe_load / yaml.safe_dump (annotated with noqa). The script is
copied to the target host via `ansible.builtin.copy` and the target
has no `utils/` package on PYTHONPATH.
noqa-annotated direct-yaml call sites (with rationale)
======================================================
* utils/cache/yaml.py — IS the cache; would recurse.
* utils/handler/yaml.py — vault-aware Loader subclass that wraps
\!vault scalars in VaultScalar; cache-by-path collapses the wrapping.
* cli/encrypt/inventory/__main__.py — vault-aware Dumper subclass that
emits \!vault literal blocks; the cached dump_yaml writes plain YAML
which is the wrong output here.
* roles/sys-svc-compose-ca/files/compose_ca.py — runtime-deployed.
* tasks/utils/migrate_meta_layout.py — once-off migration script.
Validation
==========
* Unit tests: 1573 passed.
* Lint: 56 passed (incl. new test_no_direct_yaml_calls).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
127 lines
3.7 KiB
Python
127 lines
3.7 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Iterable, List, Any
|
|
|
|
from humanfriendly import parse_size
|
|
from utils.annotations.message import warning
|
|
from utils.cache.yaml import load_yaml_any
|
|
from utils.entity_name_utils import get_entity_name
|
|
|
|
|
|
def _repo_root() -> Path:
|
|
# utils/roles/resource_validation.py -> repo root = parents[2]
|
|
return Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def _roles_root() -> Path:
|
|
return (_repo_root() / "roles").resolve()
|
|
|
|
|
|
def _deep_get(dct: dict, path: list[str]) -> Any:
|
|
cur: Any = dct
|
|
for key in path:
|
|
if not isinstance(cur, dict) or key not in cur:
|
|
return None
|
|
cur = cur[key]
|
|
return cur
|
|
|
|
|
|
def _load_yaml_file(path: Path) -> dict:
|
|
data = load_yaml_any(str(path), default_if_missing={})
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
|
|
def _parse_storage_to_gb(value: Any) -> float:
|
|
if isinstance(value, (int, float)):
|
|
return float(value)
|
|
size_bytes = parse_size(str(value))
|
|
return size_bytes / (1000**3)
|
|
|
|
|
|
def filter_roles_by_min_storage(
|
|
*,
|
|
role_names: Iterable[str],
|
|
required_storage: str | int | float,
|
|
emit_warnings: bool = False,
|
|
) -> List[str]:
|
|
roles_root_path = _roles_root()
|
|
out: List[str] = []
|
|
|
|
try:
|
|
required_gb = _parse_storage_to_gb(required_storage)
|
|
except Exception as exc:
|
|
raise ValueError(
|
|
f"Invalid required_storage value: {required_storage!r}"
|
|
) from exc
|
|
|
|
for role_name in role_names:
|
|
role_dir = (roles_root_path / role_name).resolve()
|
|
if not role_dir.is_dir():
|
|
if emit_warnings:
|
|
warning(
|
|
f"Role directory not found: {role_dir}",
|
|
title="min_storage validation",
|
|
)
|
|
continue
|
|
|
|
entity_name = get_entity_name(role_name)
|
|
if not entity_name:
|
|
if emit_warnings:
|
|
warning(
|
|
f"Could not derive entity_name from role_name '{role_name}'.",
|
|
title="min_storage validation",
|
|
)
|
|
continue
|
|
|
|
cfg_path = role_dir / "meta" / "services.yml"
|
|
if not cfg_path.is_file():
|
|
continue
|
|
|
|
try:
|
|
cfg = _load_yaml_file(cfg_path)
|
|
except Exception as exc:
|
|
if emit_warnings:
|
|
warning(
|
|
f"Failed to parse YAML: {cfg_path} ({exc})",
|
|
title="min_storage validation",
|
|
)
|
|
continue
|
|
|
|
service_cfg = _deep_get(cfg, [entity_name])
|
|
if service_cfg is None or not isinstance(service_cfg, dict):
|
|
out.append(role_name)
|
|
continue
|
|
|
|
min_storage_val = service_cfg.get("min_storage")
|
|
|
|
if min_storage_val is None:
|
|
if emit_warnings:
|
|
warning(
|
|
f"Missing key services.{entity_name}.min_storage in {cfg_path} (treating as 0GB)",
|
|
title="min_storage validation",
|
|
)
|
|
out.append(role_name)
|
|
continue
|
|
|
|
try:
|
|
min_storage_gb = _parse_storage_to_gb(min_storage_val)
|
|
except Exception as exc:
|
|
if emit_warnings:
|
|
warning(
|
|
f"Invalid min_storage value in {cfg_path}: {min_storage_val!r} ({exc})",
|
|
title="min_storage validation",
|
|
)
|
|
continue
|
|
|
|
if min_storage_gb <= required_gb:
|
|
out.append(role_name)
|
|
else:
|
|
if emit_warnings:
|
|
warning(
|
|
f"{role_name} requires {min_storage_gb:.1f}GB but runner provides only {required_gb:.1f}GB",
|
|
title="min_storage validation",
|
|
)
|
|
|
|
return out
|