mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2025-12-02 15:39:57 +00:00
Refine role dependency graph/tree builders and tests
- Refactor cli/build/graph.py to use cached metadata and dependency indices for faster graph generation and cleaner separation of concerns - Refactor cli/build/tree.py to delegate per-role processing to process_role() and support parallel execution via ProcessPoolExecutor - Add unit tests for graph helper functions and build_mappings() under tests/unit/cli/build/test_graph.py - Add unit tests for find_roles() and process_role() behaviour under tests/unit/cli/build/test_tree.py - Remove the old include_role dependency integration test which relied on the previous tree.json dependencies bucket For details see ChatGPT conversation: https://chatgpt.com/share/6926b805-28a0-800f-a075-e5250aab5c4a
This commit is contained in:
@@ -2,19 +2,76 @@
|
||||
import os
|
||||
import argparse
|
||||
import json
|
||||
from typing import Dict, Any
|
||||
from typing import Dict, Any, Optional, Iterable, Tuple
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
|
||||
from cli.build.graph import build_mappings, output_graph
|
||||
from module_utils.role_dependency_resolver import RoleDependencyResolver
|
||||
|
||||
|
||||
def find_roles(roles_dir: str):
|
||||
def find_roles(roles_dir: str) -> Iterable[Tuple[str, str]]:
|
||||
"""
|
||||
Yield (role_name, role_path) for all roles in the given roles_dir.
|
||||
"""
|
||||
for entry in os.listdir(roles_dir):
|
||||
path = os.path.join(roles_dir, entry)
|
||||
if os.path.isdir(path):
|
||||
yield entry, path
|
||||
|
||||
|
||||
def process_role(
|
||||
role_name: str,
|
||||
roles_dir: str,
|
||||
depth: int,
|
||||
shadow_folder: Optional[str],
|
||||
output: str,
|
||||
preview: bool,
|
||||
verbose: bool,
|
||||
no_include_role: bool, # currently unused, kept for CLI compatibility
|
||||
no_import_role: bool, # currently unused, kept for CLI compatibility
|
||||
no_dependencies: bool, # currently unused, kept for CLI compatibility
|
||||
no_run_after: bool, # currently unused, kept for CLI compatibility
|
||||
) -> None:
|
||||
"""
|
||||
Worker function: build graphs and (optionally) write meta/tree.json for a single role.
|
||||
|
||||
Note:
|
||||
This version no longer adds a custom top-level "dependencies" bucket.
|
||||
Only the graphs returned by build_mappings() are written.
|
||||
"""
|
||||
role_path = os.path.join(roles_dir, role_name)
|
||||
|
||||
if verbose:
|
||||
print(f"[worker] Processing role: {role_name}")
|
||||
|
||||
# Build the full graph structure (all dep types / directions) for this role
|
||||
graphs: Dict[str, Any] = build_mappings(
|
||||
start_role=role_name,
|
||||
roles_dir=roles_dir,
|
||||
max_depth=depth,
|
||||
)
|
||||
|
||||
# Preview mode: dump graphs to console instead of writing tree.json
|
||||
if preview:
|
||||
for key, data in graphs.items():
|
||||
if verbose:
|
||||
print(f"[worker] Previewing graph '{key}' for role '{role_name}'")
|
||||
# In preview mode we always output as console
|
||||
output_graph(data, "console", role_name, key)
|
||||
return
|
||||
|
||||
# Non-preview: write meta/tree.json for this role
|
||||
if shadow_folder:
|
||||
tree_file = os.path.join(shadow_folder, role_name, "meta", "tree.json")
|
||||
else:
|
||||
tree_file = os.path.join(role_path, "meta", "tree.json")
|
||||
|
||||
os.makedirs(os.path.dirname(tree_file), exist_ok=True)
|
||||
with open(tree_file, "w", encoding="utf-8") as f:
|
||||
json.dump(graphs, f, indent=2)
|
||||
|
||||
print(f"Wrote {tree_file}")
|
||||
|
||||
|
||||
def main():
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
default_roles_dir = os.path.abspath(os.path.join(script_dir, "..", "..", "roles"))
|
||||
@@ -22,24 +79,67 @@ def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate all graphs for each role and write meta/tree.json"
|
||||
)
|
||||
parser.add_argument("-d", "--role_dir", default=default_roles_dir,
|
||||
help=f"Path to roles directory (default: {default_roles_dir})")
|
||||
parser.add_argument("-D", "--depth", type=int, default=0,
|
||||
help="Max recursion depth (>0) or <=0 to stop on cycle")
|
||||
parser.add_argument("-o", "--output", choices=["yaml", "json", "console"],
|
||||
default="json", help="Output format")
|
||||
parser.add_argument("-p", "--preview", action="store_true",
|
||||
help="Preview graphs to console instead of writing files")
|
||||
parser.add_argument("-s", "--shadow-folder", type=str, default=None,
|
||||
help="If set, writes tree.json to this shadow folder instead of the role's actual meta/ folder")
|
||||
parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose logging")
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
"--role_dir",
|
||||
default=default_roles_dir,
|
||||
help=f"Path to roles directory (default: {default_roles_dir})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-D",
|
||||
"--depth",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Max recursion depth (>0) or <=0 to stop on cycle",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
choices=["yaml", "json", "console"],
|
||||
default="json",
|
||||
help="Output format for preview mode",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-p",
|
||||
"--preview",
|
||||
action="store_true",
|
||||
help="Preview graphs to console instead of writing files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-s",
|
||||
"--shadow-folder",
|
||||
type=str,
|
||||
default=None,
|
||||
help="If set, writes tree.json to this shadow folder instead of the role's actual meta/ folder",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
help="Enable verbose logging",
|
||||
)
|
||||
|
||||
# Toggles
|
||||
parser.add_argument("--no-include-role", action="store_true", help="Do not scan include_role")
|
||||
parser.add_argument("--no-import-role", action="store_true", help="Do not scan import_role")
|
||||
parser.add_argument("--no-dependencies", action="store_true", help="Do not read meta/main.yml dependencies")
|
||||
parser.add_argument("--no-run-after", action="store_true",
|
||||
help="Do not read galaxy_info.run_after from meta/main.yml")
|
||||
# Toggles (kept for CLI compatibility, currently only meaningful for future extensions)
|
||||
parser.add_argument(
|
||||
"--no-include-role",
|
||||
action="store_true",
|
||||
help="Reserved: do not include include_role in custom dependency bucket",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-import-role",
|
||||
action="store_true",
|
||||
help="Reserved: do not include import_role in custom dependency bucket",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-dependencies",
|
||||
action="store_true",
|
||||
help="Reserved: do not include meta dependencies in custom dependency bucket",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-run-after",
|
||||
action="store_true",
|
||||
help="Reserved: do not include run_after in custom dependency bucket",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -50,54 +150,53 @@ def main():
|
||||
print(f"Preview mode: {args.preview}")
|
||||
print(f"Shadow folder: {args.shadow_folder}")
|
||||
|
||||
resolver = RoleDependencyResolver(args.role_dir)
|
||||
roles = [role_name for role_name, _ in find_roles(args.role_dir)]
|
||||
|
||||
for role_name, role_path in find_roles(args.role_dir):
|
||||
if args.verbose:
|
||||
print(f"Processing role: {role_name}")
|
||||
# For preview, run sequentially to avoid completely interleaved output.
|
||||
if args.preview:
|
||||
for role_name in roles:
|
||||
process_role(
|
||||
role_name=role_name,
|
||||
roles_dir=args.role_dir,
|
||||
depth=args.depth,
|
||||
shadow_folder=args.shadow_folder,
|
||||
output=args.output,
|
||||
preview=True,
|
||||
verbose=args.verbose,
|
||||
no_include_role=args.no_include_role,
|
||||
no_import_role=args.no_import_role,
|
||||
no_dependencies=args.no_dependencies,
|
||||
no_run_after=args.no_run_after,
|
||||
)
|
||||
return
|
||||
|
||||
graphs: Dict[str, Any] = build_mappings(
|
||||
start_role=role_name,
|
||||
roles_dir=args.role_dir,
|
||||
max_depth=args.depth
|
||||
)
|
||||
# Non-preview: roles are processed in parallel
|
||||
with ProcessPoolExecutor() as executor:
|
||||
futures = {
|
||||
executor.submit(
|
||||
process_role,
|
||||
role_name,
|
||||
args.role_dir,
|
||||
args.depth,
|
||||
args.shadow_folder,
|
||||
args.output,
|
||||
False, # preview=False in parallel mode
|
||||
args.verbose,
|
||||
args.no_include_role,
|
||||
args.no_import_role,
|
||||
args.no_dependencies,
|
||||
args.no_run_after,
|
||||
): role_name
|
||||
for role_name in roles
|
||||
}
|
||||
|
||||
# Direct deps (depth=1) – getrennt erfasst für buckets
|
||||
inc_roles, imp_roles = resolver._scan_tasks(role_path)
|
||||
meta_deps = resolver._extract_meta_dependencies(role_path)
|
||||
run_after = set()
|
||||
if not args.no_run_after:
|
||||
run_after = resolver._extract_meta_run_after(role_path)
|
||||
|
||||
if any([not args.no_include_role and inc_roles,
|
||||
not args.no_import_role and imp_roles,
|
||||
not args.no_dependencies and meta_deps,
|
||||
not args.no_run_after and run_after]):
|
||||
deps_root = graphs.setdefault("dependencies", {})
|
||||
if not args.no_include_role and inc_roles:
|
||||
deps_root["include_role"] = sorted(inc_roles)
|
||||
if not args.no_import_role and imp_roles:
|
||||
deps_root["import_role"] = sorted(imp_roles)
|
||||
if not args.no_dependencies and meta_deps:
|
||||
deps_root["dependencies"] = sorted(meta_deps)
|
||||
if not args.no_run_after and run_after:
|
||||
deps_root["run_after"] = sorted(run_after)
|
||||
graphs["dependencies"] = deps_root
|
||||
|
||||
if args.preview:
|
||||
for key, data in graphs.items():
|
||||
if args.verbose:
|
||||
print(f"Previewing graph '{key}' for role '{role_name}'")
|
||||
output_graph(data, "console", role_name, key)
|
||||
else:
|
||||
if args.shadow_folder:
|
||||
tree_file = os.path.join(args.shadow_folder, role_name, "meta", "tree.json")
|
||||
else:
|
||||
tree_file = os.path.join(role_path, "meta", "tree.json")
|
||||
os.makedirs(os.path.dirname(tree_file), exist_ok=True)
|
||||
with open(tree_file, "w", encoding="utf-8") as f:
|
||||
json.dump(graphs, f, indent=2)
|
||||
print(f"Wrote {tree_file}")
|
||||
for future in as_completed(futures):
|
||||
role_name = futures[future]
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc:
|
||||
# Do not crash the whole run; report the failing role instead.
|
||||
print(f"[ERROR] Role '{role_name}' failed: {exc}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user