From 969a176be128f899054e3bade3bf739f5fca6d5b Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Tue, 20 May 2025 10:34:30 +0200 Subject: [PATCH] Optimized dependency loading --- cli/generate_playbook.py | 174 +++++++++++------- roles/docker-bigbluebutton/meta/main.yml | 4 +- roles/docker-discourse/meta/main.yml | 2 + roles/docker-keycloak/meta/main.yml | 7 +- roles/docker-lam/meta/main.yml | 2 + roles/docker-ldap/meta/main.yml | 5 +- roles/docker-mastodon/meta/main.yml | 4 +- roles/docker-matomo/meta/main.yml | 7 +- roles/docker-openproject/meta/main.yml | 3 +- roles/docker-peertube/meta/main.yml | 3 +- roles/docker-wordpress/meta/main.yml | 3 +- ...t_no_circular_before_after_dependencies.py | 78 -------- 12 files changed, 125 insertions(+), 167 deletions(-) delete mode 100644 tests/integration/test_no_circular_before_after_dependencies.py diff --git a/cli/generate_playbook.py b/cli/generate_playbook.py index 77d2072f..124a2067 100644 --- a/cli/generate_playbook.py +++ b/cli/generate_playbook.py @@ -1,13 +1,10 @@ import os -import argparse import yaml - +import argparse +from collections import defaultdict, deque def find_roles(roles_dir, prefix=None): - """ - Yield absolute paths of role directories under roles_dir. - Only include roles whose directory name starts with prefix (if given) and contain meta/main.yml. - """ + """Find all roles in the given directory.""" for entry in os.listdir(roles_dir): if prefix and not entry.startswith(prefix): continue @@ -16,87 +13,120 @@ def find_roles(roles_dir, prefix=None): if os.path.isdir(path) and os.path.isfile(meta_file): yield path, meta_file - -def load_role_order(meta_file): - """ - Load the meta/main.yml and return the role_run_order field. - Returns a dict with 'before' and 'after' keys. Defaults to empty lists if not found. - """ +def load_run_after(meta_file): + """Load the 'run_after' from the meta/main.yml of a role.""" with open(meta_file, 'r') as f: data = yaml.safe_load(f) or {} - run_order = data.get('role_run_order', {}) - before = run_order.get('before', []) - after = run_order.get('after', []) - - # If "all" is in before or after, treat it as a special value - if "all" in before: - before.remove("all") - before.insert(0, "all") # Treat "all" as the first item - if "all" in after: - after.remove("all") - after.append("all") # Treat "all" as the last item - - return { - 'before': before, - 'after': after - } + return data.get('galaxy_info', {}).get('run_after', []) +def load_application_id(role_path): + """Load the application_id from the vars/main.yml of the role.""" + vars_file = os.path.join(role_path, 'vars', 'main.yml') + if os.path.exists(vars_file): + with open(vars_file, 'r') as f: + data = yaml.safe_load(f) or {} + return data.get('application_id') + return None + +def build_dependency_graph(roles_dir, prefix=None): + """Build a dependency graph where each role points to the roles it depends on.""" + graph = defaultdict(list) + in_degree = defaultdict(int) + roles = {} -def sort_roles_by_order(roles_dir, prefix=None): - roles = [] - - # Collect roles and their before/after dependencies for role_path, meta_file in find_roles(roles_dir, prefix): - run_order = load_role_order(meta_file) + run_after = load_run_after(meta_file) + application_id = load_application_id(role_path) role_name = os.path.basename(role_path) - roles.append({ + roles[role_name] = { 'role_name': role_name, - 'before': run_order['before'], - 'after': run_order['after'], + 'run_after': run_after, + 'application_id': application_id, 'path': role_path - }) + } - # Now sort the roles based on before/after relationships + # If the role has dependencies, build the graph + for dependency in run_after: + graph[dependency].append(role_name) + in_degree[role_name] += 1 + + # Ensure roles with no dependencies have an in-degree of 0 + if role_name not in in_degree: + in_degree[role_name] = 0 + + return graph, in_degree, roles + +def topological_sort(graph, in_degree): + """Perform topological sort on the dependency graph.""" + # Queue for roles with no incoming dependencies (in_degree == 0) + queue = deque([role for role, degree in in_degree.items() if degree == 0]) sorted_roles = [] - unresolved_roles = roles[:] - - # First, place roles with "before: all" at the start - roles_with_before_all = [role for role in unresolved_roles if "all" in role['before']] - sorted_roles.extend(roles_with_before_all) - unresolved_roles = [role for role in unresolved_roles if "all" not in role['before']] - - while unresolved_roles: - # Find roles with no dependencies in 'before' - ready_roles = [role for role in unresolved_roles if not any(dep in [r['role_name'] for r in unresolved_roles] for dep in role['before'])] - - if not ready_roles: - raise ValueError("Circular dependency detected in 'before'/'after' fields") - for role in ready_roles: - sorted_roles.append(role) - unresolved_roles.remove(role) + while queue: + role = queue.popleft() + sorted_roles.append(role) - # Remove from the 'before' lists of remaining roles - for r in unresolved_roles: - r['before'] = [dep for dep in r['before'] if dep != role['role_name']] + # Reduce in-degree for roles dependent on the current role + for neighbor in graph[role]: + in_degree[neighbor] -= 1 + if in_degree[neighbor] == 0: + queue.append(neighbor) - # Finally, place roles with "after: all" at the end - roles_with_after_all = [role for role in unresolved_roles if "all" in role['after']] - sorted_roles.extend(roles_with_after_all) - unresolved_roles = [role for role in unresolved_roles if "all" not in role['after']] + if len(sorted_roles) != len(in_degree): + # If the number of sorted roles doesn't match the number of roles, + # there was a cycle in the graph (not all roles could be sorted) + raise Exception("Circular dependency detected among the roles!") return sorted_roles +def print_dependency_tree(graph): + """Print the dependency tree visually on the console.""" + def print_node(role, indent=0): + print(" " * indent + role) + for dependency in graph[role]: + print_node(dependency, indent + 1) + + # Print the tree starting from roles with no dependencies + all_roles = set(graph.keys()) + dependent_roles = {role for dependencies in graph.values() for role in dependencies} + root_roles = all_roles - dependent_roles + + printed_roles = [] + + def collect_roles(role, indent=0): + printed_roles.append(role) + for dependency in graph[role]: + collect_roles(dependency, indent + 1) + + for root in root_roles: + collect_roles(root) + + return printed_roles def generate_playbook_entries(roles_dir, prefix=None): - entries = [] - sorted_roles = sort_roles_by_order(roles_dir, prefix) + """Generate playbook entries based on the sorted order.""" + # Build dependency graph + graph, in_degree, roles = build_dependency_graph(roles_dir, prefix) - for role in sorted_roles: - # entry text + # Print and collect roles in tree order + tree_sorted_roles = print_dependency_tree(graph) + + # Topologically sort the roles + sorted_role_names = topological_sort(graph, in_degree) + + # Ensure that roles that appear in the tree come first + final_sorted_roles = [role for role in tree_sorted_roles if role in sorted_role_names] + + # Include the remaining unsorted roles + final_sorted_roles += [role for role in sorted_role_names if role not in final_sorted_roles] + + # Generate the playbook entries + entries = [] + for role_name in final_sorted_roles: + role = roles[role_name] entry = ( - f"- name: setup {role['role_name']}\n" - f" when: (\"{role['role_name']}\" in group_names)\n" + f"- name: setup {role['application_id']}\n" # Use application_id here + f" when: ('{role['application_id']}' in group_names)\n" # Correct condition format f" include_role:\n" f" name: {role['role_name']}\n" ) @@ -104,10 +134,9 @@ def generate_playbook_entries(roles_dir, prefix=None): return entries - def main(): parser = argparse.ArgumentParser( - description='Generate an Ansible playbook include file from Docker roles and application_ids, sorted by role_run_order.' + description='Generate an Ansible playbook include file from Docker roles, sorted by run_after order.' ) parser.add_argument( 'roles_dir', @@ -123,8 +152,14 @@ def main(): help='Output file path (default: stdout)', default=None ) + parser.add_argument( + '-t', '--tree', + action='store_true', + help='Display the dependency tree of roles visually' + ) args = parser.parse_args() + # Generate and output the playbook entries entries = generate_playbook_entries(args.roles_dir, args.prefix) output = ''.join(entries) @@ -135,6 +170,5 @@ def main(): else: print(output) - if __name__ == '__main__': main() diff --git a/roles/docker-bigbluebutton/meta/main.yml b/roles/docker-bigbluebutton/meta/main.yml index 6688dbfd..6a0a0e45 100644 --- a/roles/docker-bigbluebutton/meta/main.yml +++ b/roles/docker-bigbluebutton/meta/main.yml @@ -28,4 +28,6 @@ galaxy_info: issue_tracker_url: https://s.veen.world/cymaisissues documentation: https://s.veen.world/cymais logo: - class: "fa-solid fa-chalkboard-teacher" \ No newline at end of file + class: "fa-solid fa-chalkboard-teacher" + run_after: + - docker-keycloak \ No newline at end of file diff --git a/roles/docker-discourse/meta/main.yml b/roles/docker-discourse/meta/main.yml index a4f8b979..5804aad4 100644 --- a/roles/docker-discourse/meta/main.yml +++ b/roles/docker-discourse/meta/main.yml @@ -19,4 +19,6 @@ galaxy_info: documentation: https://s.veen.world/cymais logo: class: "fa-solid fa-comments" + run_after: + - docker-wordpress dependencies: [] diff --git a/roles/docker-keycloak/meta/main.yml b/roles/docker-keycloak/meta/main.yml index 86587eb0..bf1b1a78 100644 --- a/roles/docker-keycloak/meta/main.yml +++ b/roles/docker-keycloak/meta/main.yml @@ -19,9 +19,6 @@ galaxy_info: documentation: https://s.veen.world/cymais logo: class: "fa-solid fa-lock" + run_after: + - docker-matomo dependencies: [] -role_run_order: - before: - - all - after: - - docker-ldap diff --git a/roles/docker-lam/meta/main.yml b/roles/docker-lam/meta/main.yml index c35dabbe..b95b60ac 100644 --- a/roles/docker-lam/meta/main.yml +++ b/roles/docker-lam/meta/main.yml @@ -19,4 +19,6 @@ galaxy_info: documentation: https://s.veen.world/cymais logo: class: "fa-solid fa-network-wired" + run_after: + - docker-keycloak dependencies: [] diff --git a/roles/docker-ldap/meta/main.yml b/roles/docker-ldap/meta/main.yml index 227cc4bc..f3796361 100644 --- a/roles/docker-ldap/meta/main.yml +++ b/roles/docker-ldap/meta/main.yml @@ -20,7 +20,6 @@ galaxy_info: documentation: https://s.veen.world/cymais logo: class: "fa-solid fa-users" + #run_after: + # - "0" dependencies: [] -role_run_order: - before: - - all diff --git a/roles/docker-mastodon/meta/main.yml b/roles/docker-mastodon/meta/main.yml index 07f1322c..92c35117 100644 --- a/roles/docker-mastodon/meta/main.yml +++ b/roles/docker-mastodon/meta/main.yml @@ -21,5 +21,5 @@ galaxy_info: documentation: "https://s.veen.world/cymais" logo: class: "fa-solid fa-bullhorn" -role_run_order: - after: docker-keycloak \ No newline at end of file + run_after: + - docker-keycloak \ No newline at end of file diff --git a/roles/docker-matomo/meta/main.yml b/roles/docker-matomo/meta/main.yml index 6f53a3ce..0aa4ebe4 100644 --- a/roles/docker-matomo/meta/main.yml +++ b/roles/docker-matomo/meta/main.yml @@ -18,8 +18,5 @@ galaxy_info: documentation: "https://s.veen.world/cymais" logo: class: "fa-solid fa-chart-line" -role_run_order: - before: - - all - after: - - docker-keycloak \ No newline at end of file + run_after: + - "docker-ldap" \ No newline at end of file diff --git a/roles/docker-openproject/meta/main.yml b/roles/docker-openproject/meta/main.yml index bba729d3..0abb2e9c 100644 --- a/roles/docker-openproject/meta/main.yml +++ b/roles/docker-openproject/meta/main.yml @@ -29,4 +29,5 @@ galaxy_info: documentation: "https://s.veen.world/cymais" logo: class: "fa-solid fa-project-diagram" -dependencies: [] + run_after: + - docker-keycloak diff --git a/roles/docker-peertube/meta/main.yml b/roles/docker-peertube/meta/main.yml index 48a56d74..61c0aac4 100644 --- a/roles/docker-peertube/meta/main.yml +++ b/roles/docker-peertube/meta/main.yml @@ -28,4 +28,5 @@ galaxy_info: documentation: "https://s.veen.world/cymais" logo: class: "fa-solid fa-video" - dependencies: [] + run_after: + - docker-keycloak diff --git a/roles/docker-wordpress/meta/main.yml b/roles/docker-wordpress/meta/main.yml index 4036bd0b..d9816f59 100644 --- a/roles/docker-wordpress/meta/main.yml +++ b/roles/docker-wordpress/meta/main.yml @@ -27,4 +27,5 @@ galaxy_info: documentation: "https://s.veen.world/cymais" logo: class: "fa-solid fa-blog" -dependencies: [] + run_after: + - docker-keycloak \ No newline at end of file diff --git a/tests/integration/test_no_circular_before_after_dependencies.py b/tests/integration/test_no_circular_before_after_dependencies.py deleted file mode 100644 index caebd63d..00000000 --- a/tests/integration/test_no_circular_before_after_dependencies.py +++ /dev/null @@ -1,78 +0,0 @@ -import unittest -import os -import yaml - -def load_yaml_file(file_path): - """Load a YAML file and return its content.""" - with open(file_path, 'r') as file: - return yaml.safe_load(file) or {} - -def get_meta_info(role_path): - """Extract before and after dependencies from the meta/main.yml of a role.""" - meta_file = os.path.join(role_path, 'meta', 'main.yml') - if not os.path.isfile(meta_file): - return [], [] - meta_data = load_yaml_file(meta_file) - run_order = meta_data.get('role_run_order', {}) - before = run_order.get('before', []) - after = run_order.get('after', []) - return before, after - -def resolve_dependencies(roles_dir): - """Resolve all role dependencies and detect circular dependencies in before/after.""" - visited = set() # Tracks roles that have been processed - stack = set() # Tracks roles being processed in the current recursion path - - def visit(role_path, stack): - role_name = os.path.basename(role_path) - - # Check for circular dependencies in the recursion path (for before/after) - if role_name in stack: - raise ValueError(f"Circular dependency detected in 'before'/'after' between roles: {' -> '.join(stack)} -> {role_name}") - - # Check if role is already processed - if role_name in visited: - return [] - - # Mark role as visited and add to stack - visited.add(role_name) - stack.append(role_name) - - # Get before and after dependencies - before, after = get_meta_info(role_path) - - # Resolve before and after roles - for before_role in before: - before_role_path = os.path.join(roles_dir, before_role) - visit(before_role_path, stack) # Recurse into before dependencies - - for after_role in after: - after_role_path = os.path.join(roles_dir, after_role) - visit(after_role_path, stack) # Recurse into after dependencies - - stack.pop() # Remove the current role from the stack - - for role_name in os.listdir(roles_dir): - role_path = os.path.join(roles_dir, role_name) - if os.path.isdir(role_path): - try: - visit(role_path, []) # Start recursion from this role - except ValueError as e: - raise ValueError(f"Error processing role '{role_name}' at path '{role_path}': {str(e)}") - - return "No Circular Dependency Detected" - -class TestRoleBeforeAfterDependencies(unittest.TestCase): - def test_no_circular_before_after_dependencies(self): - roles_dir = "roles" # Path to the roles directory - - try: - result = resolve_dependencies(roles_dir) - except ValueError as e: - self.fail(f"Circular dependency detected: {e}") - - # If no exception, the test passed - self.assertEqual(result, "No Circular Dependency Detected") - -if __name__ == '__main__': - unittest.main()