mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2025-08-29 23:08:06 +02:00
Refactor and extend role dependency resolution:
- Introduced module_utils/role_dependency_resolver.py with full support for include_role, import_role, meta dependencies, and run_after. - Refactored cli/build/tree.py to use RoleDependencyResolver (added toggles for include/import/dependencies/run_after). - Extended filter_plugins/canonical_domains_map.py with optional 'recursive' mode (ignores run_after by design). - Updated roles/web-app-nextcloud to properly include Collabora dependency. - Added comprehensive unittests under tests/unit/module_utils for RoleDependencyResolver. Ref: https://chatgpt.com/share/68a519c8-8e54-800f-83c0-be38546620d9
This commit is contained in:
@@ -2,174 +2,45 @@
|
||||
import os
|
||||
import argparse
|
||||
import json
|
||||
import fnmatch
|
||||
import re
|
||||
from typing import Dict, Any
|
||||
|
||||
import yaml
|
||||
|
||||
from cli.build.graph import build_mappings, output_graph
|
||||
from module_utils.role_dependency_resolver import RoleDependencyResolver
|
||||
|
||||
|
||||
def find_roles(roles_dir: str):
|
||||
"""Yield (role_name, role_path) for every subfolder in roles_dir."""
|
||||
for entry in os.listdir(roles_dir):
|
||||
path = os.path.join(roles_dir, entry)
|
||||
if os.path.isdir(path):
|
||||
yield entry, path
|
||||
|
||||
|
||||
def _is_pure_jinja_var(s: str) -> bool:
|
||||
"""Check if string is exactly a single {{ var }} expression."""
|
||||
return bool(re.fullmatch(r"\s*\{\{\s*[^}]+\s*\}\}\s*", s))
|
||||
|
||||
|
||||
def _jinja_to_glob(s: str) -> str:
|
||||
"""Convert Jinja placeholders {{ ... }} into * for fnmatch."""
|
||||
pattern = re.sub(r"\{\{[^}]+\}\}", "*", s)
|
||||
pattern = re.sub(r"\*{2,}", "*", pattern)
|
||||
return pattern.strip()
|
||||
|
||||
|
||||
def _list_role_dirs(roles_dir: str) -> list[str]:
|
||||
"""Return a list of role directory names inside roles_dir."""
|
||||
return [
|
||||
d for d in os.listdir(roles_dir)
|
||||
if os.path.isdir(os.path.join(roles_dir, d))
|
||||
]
|
||||
|
||||
|
||||
def find_include_role_dependencies(role_path: str, roles_dir: str) -> set[str]:
|
||||
"""
|
||||
Scan all tasks/*.yml(.yaml) files of a role and collect include_role dependencies.
|
||||
|
||||
Rules:
|
||||
- loop/with_items with literal strings -> add those as roles
|
||||
- name contains jinja AND surrounding literals -> convert to glob and match existing roles
|
||||
- name is a pure jinja variable only -> ignore
|
||||
- name is a pure literal -> add as-is
|
||||
"""
|
||||
deps: set[str] = set()
|
||||
tasks_dir = os.path.join(role_path, "tasks")
|
||||
if not os.path.isdir(tasks_dir):
|
||||
return deps
|
||||
|
||||
candidates = []
|
||||
for root, _, files in os.walk(tasks_dir):
|
||||
for f in files:
|
||||
if f.endswith(".yml") or f.endswith(".yaml"):
|
||||
candidates.append(os.path.join(root, f))
|
||||
|
||||
all_roles = _list_role_dirs(roles_dir)
|
||||
|
||||
def add_literal_loop_items(loop_val):
|
||||
if isinstance(loop_val, list):
|
||||
for item in loop_val:
|
||||
if isinstance(item, str) and item.strip():
|
||||
deps.add(item.strip())
|
||||
|
||||
for file_path in candidates:
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
docs = list(yaml.safe_load_all(f))
|
||||
except Exception:
|
||||
# Be tolerant to any parsing issues; skip unreadable files
|
||||
continue
|
||||
|
||||
for doc in docs:
|
||||
if not isinstance(doc, list):
|
||||
continue
|
||||
for task in doc:
|
||||
if not isinstance(task, dict):
|
||||
continue
|
||||
if "include_role" not in task:
|
||||
continue
|
||||
inc = task.get("include_role")
|
||||
if not isinstance(inc, dict):
|
||||
continue
|
||||
name = inc.get("name")
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
continue
|
||||
|
||||
# 1) Handle loop/with_items
|
||||
loop_val = task.get("loop", task.get("with_items"))
|
||||
if loop_val is not None:
|
||||
add_literal_loop_items(loop_val)
|
||||
# still check name for surrounding literals
|
||||
if not _is_pure_jinja_var(name):
|
||||
pattern = (
|
||||
_jinja_to_glob(name)
|
||||
if ("{{" in name and "}}" in name)
|
||||
else name
|
||||
)
|
||||
if "*" in pattern:
|
||||
for r in all_roles:
|
||||
if fnmatch.fnmatch(r, pattern):
|
||||
deps.add(r)
|
||||
continue
|
||||
|
||||
# 2) No loop: evaluate name
|
||||
if "{{" in name and "}}" in name:
|
||||
if _is_pure_jinja_var(name):
|
||||
continue # ignore pure variable
|
||||
pattern = _jinja_to_glob(name)
|
||||
if "*" in pattern:
|
||||
for r in all_roles:
|
||||
if fnmatch.fnmatch(r, pattern):
|
||||
deps.add(r)
|
||||
continue
|
||||
else:
|
||||
deps.add(pattern)
|
||||
else:
|
||||
# pure literal
|
||||
deps.add(name.strip())
|
||||
|
||||
return deps
|
||||
|
||||
|
||||
def main():
|
||||
# default roles dir is ../../roles relative to this script
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
default_roles_dir = os.path.abspath(
|
||||
os.path.join(script_dir, "..", "..", "roles")
|
||||
)
|
||||
default_roles_dir = os.path.abspath(os.path.join(script_dir, "..", "..", "roles"))
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate all graphs for each role and write meta/tree.json"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-d", "--role_dir",
|
||||
default=default_roles_dir,
|
||||
help=f"Path to roles directory (default: {default_roles_dir})"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-D", "--depth",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Max recursion depth (>0) or <=0 to stop on cycle"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o", "--output",
|
||||
choices=["yaml", "json", "console"],
|
||||
default="json",
|
||||
help="Output format"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-p", "--preview",
|
||||
action="store_true",
|
||||
help="Preview graphs to console instead of writing files"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-s", "--shadow-folder",
|
||||
type=str,
|
||||
default=None,
|
||||
help="If set, writes tree.json to this shadow folder instead of the role's actual meta/ folder"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v", "--verbose",
|
||||
action="store_true",
|
||||
help="Enable verbose logging"
|
||||
)
|
||||
parser.add_argument("-d", "--role_dir", default=default_roles_dir,
|
||||
help=f"Path to roles directory (default: {default_roles_dir})")
|
||||
parser.add_argument("-D", "--depth", type=int, default=0,
|
||||
help="Max recursion depth (>0) or <=0 to stop on cycle")
|
||||
parser.add_argument("-o", "--output", choices=["yaml", "json", "console"],
|
||||
default="json", help="Output format")
|
||||
parser.add_argument("-p", "--preview", action="store_true",
|
||||
help="Preview graphs to console instead of writing files")
|
||||
parser.add_argument("-s", "--shadow-folder", type=str, default=None,
|
||||
help="If set, writes tree.json to this shadow folder instead of the role's actual meta/ folder")
|
||||
parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose logging")
|
||||
|
||||
# Toggles
|
||||
parser.add_argument("--no-include-role", action="store_true", help="Do not scan include_role")
|
||||
parser.add_argument("--no-import-role", action="store_true", help="Do not scan import_role")
|
||||
parser.add_argument("--no-dependencies", action="store_true", help="Do not read meta/main.yml dependencies")
|
||||
parser.add_argument("--no-run-after", action="store_true",
|
||||
help="Do not read galaxy_info.run_after from meta/main.yml")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
@@ -179,6 +50,8 @@ def main():
|
||||
print(f"Preview mode: {args.preview}")
|
||||
print(f"Shadow folder: {args.shadow_folder}")
|
||||
|
||||
resolver = RoleDependencyResolver(args.role_dir)
|
||||
|
||||
for role_name, role_path in find_roles(args.role_dir):
|
||||
if args.verbose:
|
||||
print(f"Processing role: {role_name}")
|
||||
@@ -189,13 +62,26 @@ def main():
|
||||
max_depth=args.depth
|
||||
)
|
||||
|
||||
# add include_role dependencies from tasks
|
||||
include_deps = find_include_role_dependencies(role_path, args.role_dir)
|
||||
if include_deps:
|
||||
# Direct deps (depth=1) – getrennt erfasst für buckets
|
||||
inc_roles, imp_roles = resolver._scan_tasks(role_path)
|
||||
meta_deps = resolver._extract_meta_dependencies(role_path)
|
||||
run_after = set()
|
||||
if not args.no_run_after:
|
||||
run_after = resolver._extract_meta_run_after(role_path)
|
||||
|
||||
if any([not args.no_include_role and inc_roles,
|
||||
not args.no_import_role and imp_roles,
|
||||
not args.no_dependencies and meta_deps,
|
||||
not args.no_run_after and run_after]):
|
||||
deps_root = graphs.setdefault("dependencies", {})
|
||||
inc_list = set(deps_root.get("include_role", []))
|
||||
inc_list.update(include_deps)
|
||||
deps_root["include_role"] = sorted(inc_list)
|
||||
if not args.no_include_role and inc_roles:
|
||||
deps_root["include_role"] = sorted(inc_roles)
|
||||
if not args.no_import_role and imp_roles:
|
||||
deps_root["import_role"] = sorted(imp_roles)
|
||||
if not args.no_dependencies and meta_deps:
|
||||
deps_root["dependencies"] = sorted(meta_deps)
|
||||
if not args.no_run_after and run_after:
|
||||
deps_root["run_after"] = sorted(run_after)
|
||||
graphs["dependencies"] = deps_root
|
||||
|
||||
if args.preview:
|
||||
@@ -205,13 +91,11 @@ def main():
|
||||
output_graph(data, "console", role_name, key)
|
||||
else:
|
||||
if args.shadow_folder:
|
||||
tree_file = os.path.join(
|
||||
args.shadow_folder, role_name, "meta", "tree.json"
|
||||
)
|
||||
tree_file = os.path.join(args.shadow_folder, role_name, "meta", "tree.json")
|
||||
else:
|
||||
tree_file = os.path.join(role_path, "meta", "tree.json")
|
||||
os.makedirs(os.path.dirname(tree_file), exist_ok=True)
|
||||
with open(tree_file, "w") as f:
|
||||
with open(tree_file, "w", encoding="utf-8") as f:
|
||||
json.dump(graphs, f, indent=2)
|
||||
print(f"Wrote {tree_file}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user