mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 02:54:26 +00:00
refactor(deploy): resolve role groups at runtime instead of generating task files
`cli build include` generated one tasks/groups/<group>-roles.yml per invokable category during setup, and the stage plays included those files. They were written by the in-container UID, which is why a dedicated clean-container-owned helper had to delete them from inside the container.
The stage plays now loop over lookup('group_roles', <group>) and include tasks/utils/setup/{group,role}.yml directly. ordered_roles() in utils/roles/order.py holds the run_after topological sort the generator used to perform at build time.
utils/roles/stage.py:_group_names() no longer globs the generated files; it reads get_invokable_paths(), the same categories.yml source those files were derived from.
Measured: handlers still flush after every role (ansible-core 2.20.2, three loop items, three separate handler runs), and resolving all 18 groups costs 0.60s cold against 1.4s for the generator.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
kevinveenbirkenbachandClaude Opus 5 committed
1 parent
3b4bc4ee65
commit
351b64e2c5
24 files changed
+363
-463
No files matched your search
@@ -118,7 +118,7 @@ clean-cache:
|
||||
@bash scripts/system/cache/clean.sh
|
||||
|
||||
.PHONY: clean-container-owned
|
||||
# Remove container-owned generated artefacts (build/, tasks/groups/*.yml).
|
||||
# Remove container-owned generated artefacts (build/).
|
||||
# Note: these files are created inside the compose container with the in-container UID (typically `nobody`); the host cannot rm them directly.
|
||||
# Note: the helper auto-starts a stopped infinito container before deleting; safe no-op when the targets do not exist.
|
||||
clean-container-owned:
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
"""Compatibility wrapper.
|
||||
|
||||
The implementation lives in ``cli/build/include/__main__.py``; this package
|
||||
re-exports its public API so ``from cli.build.include import ...`` works.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from . import __main__ as _main
|
||||
__all__ = getattr(_main, "__all__", [n for n in dir(_main) if not n.startswith("_")])
|
||||
globals().update({name: getattr(_main, name) for name in __all__})
|
||||
|
||||
# Prefer explicit __all__ if the original module defined it.
|
||||
__all__ = getattr(_main, "__all__", [n for n in dir(_main) if not n.startswith("_")])
|
||||
@@ -1,239 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
from collections import defaultdict, deque
|
||||
|
||||
from utils.cache.yaml import load_yaml
|
||||
from utils.roles.mapping import ROLE_FILE_META_MAIN, ROLE_FILE_VARS_MAIN
|
||||
|
||||
def find_roles(roles_dir, prefixes=None):
|
||||
"""
|
||||
Find all roles in the given directory whose names start with
|
||||
any of the provided prefixes. If prefixes is empty or None,
|
||||
include all roles.
|
||||
"""
|
||||
for entry in os.listdir(roles_dir):
|
||||
if prefixes:
|
||||
if not any(entry.startswith(pref) or entry == pref.rstrip("-") for pref in prefixes):
|
||||
continue
|
||||
path = os.path.join(roles_dir, entry)
|
||||
meta_file = os.path.join(path, ROLE_FILE_META_MAIN)
|
||||
if os.path.isdir(path) and os.path.isfile(meta_file):
|
||||
yield path, meta_file
|
||||
|
||||
def load_run_after(meta_file):
|
||||
from utils.roles.meta_lookup import get_role_run_after
|
||||
|
||||
role_path = os.path.abspath(os.path.dirname(os.path.dirname(meta_file)))
|
||||
role_name = os.path.basename(role_path)
|
||||
try:
|
||||
return get_role_run_after(role_path, role_name=role_name)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def load_application_id(role_path):
|
||||
"""Load the application_id from the vars/main.yml of the role."""
|
||||
vars_file = os.path.join(role_path, ROLE_FILE_VARS_MAIN)
|
||||
if os.path.exists(vars_file):
|
||||
data = load_yaml(vars_file)
|
||||
return data.get('application_id')
|
||||
return None
|
||||
|
||||
def build_dependency_graph(roles_dir, prefixes=None):
|
||||
"""
|
||||
Build a dependency graph where each key is a role name and
|
||||
its value is a list of roles that depend on it.
|
||||
Also return in_degree counts and the roles metadata map.
|
||||
"""
|
||||
graph = defaultdict(list)
|
||||
in_degree = defaultdict(int)
|
||||
roles = {}
|
||||
|
||||
in_scope: set[str] = set()
|
||||
role_entries = list(find_roles(roles_dir, prefixes))
|
||||
for role_path, _meta_file in role_entries:
|
||||
in_scope.add(os.path.basename(role_path))
|
||||
|
||||
for role_path, meta_file in role_entries:
|
||||
run_after = load_run_after(meta_file)
|
||||
application_id = load_application_id(role_path)
|
||||
role_name = os.path.basename(role_path)
|
||||
|
||||
in_scope_deps = [d for d in run_after if d in in_scope]
|
||||
|
||||
roles[role_name] = {
|
||||
'role_name': role_name,
|
||||
'run_after': in_scope_deps,
|
||||
'application_id': application_id,
|
||||
'path': role_path
|
||||
}
|
||||
|
||||
for dependency in in_scope_deps:
|
||||
graph[dependency].append(role_name)
|
||||
in_degree[role_name] += 1
|
||||
|
||||
if role_name not in in_degree:
|
||||
in_degree[role_name] = 0
|
||||
|
||||
return graph, in_degree, roles
|
||||
|
||||
def find_cycle(roles):
|
||||
"""
|
||||
Detect a cycle in the run_after relations:
|
||||
roles: dict mapping role_name -> { 'run_after': [...], ... }
|
||||
Returns a list of role_names forming the cycle (with the start repeated at end), or None.
|
||||
"""
|
||||
visited = set()
|
||||
stack = set()
|
||||
|
||||
def dfs(node, path):
|
||||
visited.add(node)
|
||||
stack.add(node)
|
||||
path.append(node)
|
||||
for dep in roles.get(node, {}).get('run_after', []):
|
||||
if dep not in visited:
|
||||
res = dfs(dep, path)
|
||||
if res:
|
||||
return res
|
||||
elif dep in stack:
|
||||
idx = path.index(dep)
|
||||
return path[idx:] + [dep]
|
||||
stack.remove(node)
|
||||
path.pop()
|
||||
return None
|
||||
|
||||
for role in roles:
|
||||
if role not in visited:
|
||||
cycle = dfs(role, [])
|
||||
if cycle:
|
||||
return cycle
|
||||
return None
|
||||
|
||||
def topological_sort(graph, in_degree, roles=None):
|
||||
"""
|
||||
Perform topological sort on the dependency graph.
|
||||
If a cycle is detected, raise an Exception with detailed debug info.
|
||||
"""
|
||||
|
||||
queue = deque([r for r, d in in_degree.items() if d == 0])
|
||||
sorted_roles = []
|
||||
local_in = dict(in_degree)
|
||||
|
||||
while queue:
|
||||
role = queue.popleft()
|
||||
sorted_roles.append(role)
|
||||
for nbr in graph.get(role, []):
|
||||
local_in[nbr] -= 1
|
||||
if local_in[nbr] == 0:
|
||||
queue.append(nbr)
|
||||
|
||||
if len(sorted_roles) != len(in_degree):
|
||||
cycle = find_cycle(roles or {})
|
||||
unsorted = [r for r in in_degree if r not in sorted_roles]
|
||||
|
||||
header = "❌ Dependency resolution failed"
|
||||
if cycle:
|
||||
reason = f"Circular dependency detected: {' -> '.join(cycle)}"
|
||||
else:
|
||||
reason = "Unresolved dependencies among roles (possible cycle or missing role)."
|
||||
|
||||
details = []
|
||||
if unsorted:
|
||||
details.append("Unsorted roles and their declared run_after dependencies:")
|
||||
for r in unsorted:
|
||||
deps = roles.get(r, {}).get('run_after', [])
|
||||
details.append(f" - {r} depends on {deps!r}")
|
||||
|
||||
graph_repr = f"Full dependency graph: {dict(graph)!r}"
|
||||
|
||||
raise Exception("\n".join([header, reason] + details + [graph_repr]))
|
||||
|
||||
return sorted_roles
|
||||
|
||||
def print_dependency_tree(graph):
|
||||
"""Print the dependency tree visually on the console."""
|
||||
def print_node(role, indent=0):
|
||||
print(" " * indent + role)
|
||||
for dep in graph.get(role, []):
|
||||
print_node(dep, indent + 1)
|
||||
|
||||
all_roles = set(graph.keys())
|
||||
dependent = {r for deps in graph.values() for r in deps}
|
||||
roots = all_roles - dependent
|
||||
|
||||
for root in roots:
|
||||
print_node(root)
|
||||
|
||||
def gen_condi_role_incl(roles_dir, prefixes=None):
|
||||
"""
|
||||
Generate playbook entries based on the sorted order.
|
||||
Raises a ValueError if application_id is missing.
|
||||
"""
|
||||
graph, in_degree, roles = build_dependency_graph(roles_dir, prefixes)
|
||||
sorted_names = topological_sort(graph, in_degree, roles)
|
||||
|
||||
entries = []
|
||||
for role_name in sorted_names:
|
||||
role = roles[role_name]
|
||||
|
||||
if role.get('application_id') is None:
|
||||
vars_file = os.path.join(role['path'], ROLE_FILE_VARS_MAIN)
|
||||
raise ValueError(f"'application_id' missing in {vars_file}")
|
||||
|
||||
app_id = role['application_id']
|
||||
entries.append(
|
||||
f"- name: 🔧 setup {app_id}\n"
|
||||
f" when: ('{app_id}' | application_allowed(group_names, lookup('deployment').whitelist))\n"
|
||||
f" include_role:\n"
|
||||
f" name: {role_name}\n"
|
||||
)
|
||||
entries.append(
|
||||
f"- name: 🚿 flush handlers after {app_id}\n"
|
||||
f" meta: flush_handlers\n"
|
||||
)
|
||||
|
||||
return entries
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Generate an Ansible playbook include file from Docker roles, sorted by run_after order.'
|
||||
)
|
||||
parser.add_argument('roles_dir', help='Path to directory containing role folders')
|
||||
parser.add_argument(
|
||||
'-p', '--prefix',
|
||||
action='append',
|
||||
help='Only include roles whose names start with any of these prefixes; can be specified multiple times'
|
||||
)
|
||||
parser.add_argument('-o', '--output', default=None,
|
||||
help='Output file path (default: stdout)')
|
||||
parser.add_argument('-t', '--tree', action='store_true',
|
||||
help='Display the dependency tree of roles and exit')
|
||||
|
||||
args = parser.parse_args()
|
||||
prefixes = args.prefix or []
|
||||
|
||||
if args.tree:
|
||||
graph, _, _ = build_dependency_graph(args.roles_dir, prefixes)
|
||||
print_dependency_tree(graph)
|
||||
sys.exit(0)
|
||||
|
||||
entries = gen_condi_role_incl(args.roles_dir, prefixes)
|
||||
output = ''.join(entries)
|
||||
|
||||
if args.output:
|
||||
os.makedirs(os.path.dirname(args.output), exist_ok=True)
|
||||
if os.path.exists(args.output) and not os.access(args.output, os.W_OK):
|
||||
try:
|
||||
os.unlink(args.output)
|
||||
except OSError:
|
||||
pass
|
||||
with open(args.output, 'w') as f:
|
||||
f.write(output)
|
||||
print(f"Playbook entries written to {args.output}")
|
||||
else:
|
||||
print(output)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,3 +1,3 @@
|
||||
# Order 📜
|
||||
|
||||
Show role execution order from both scheduling layers: `run` prints the group-file call order of the main pass (`tasks/groups/*.yml`), `preload` prints the sys-service-loader preload order (run_after topological sort over the service registry).
|
||||
Show role execution order from both scheduling layers: `run` prints the group call order of the main pass, `preload` prints the sys-service-loader preload order (run_after topological sort over the service registry).
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Lookup `group_roles`: the roles of a role group in run_after order.
|
||||
|
||||
Returns one ``{'role': …, 'app': …}`` mapping per role, so a stage can loop
|
||||
over the group instead of including a generated task file per group:
|
||||
|
||||
loop: "{{ lookup('group_roles', 'web-app') }}"
|
||||
|
||||
`role` is the directory under ``roles/``, `app` the role's application_id --
|
||||
the value the deployment whitelist and `application_allowed` speak in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ansible.errors import AnsibleError
|
||||
from ansible.plugins.lookup import LookupBase
|
||||
|
||||
from utils.roles.order import ordered_roles
|
||||
|
||||
|
||||
class LookupModule(LookupBase):
|
||||
def run(
|
||||
self,
|
||||
terms: list[Any] | None,
|
||||
variables: dict[str, Any] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> list[list[dict[str, str]]]:
|
||||
if not terms or len(terms) != 1:
|
||||
raise AnsibleError(
|
||||
"group_roles lookup requires exactly one term: the group"
|
||||
)
|
||||
|
||||
group = str(terms[0]).strip()
|
||||
if not group:
|
||||
raise AnsibleError("group_roles lookup: the group must not be empty")
|
||||
|
||||
roles_dir = str(kwargs.get("roles_dir") or Path.cwd() / "roles")
|
||||
try:
|
||||
return [[dict(entry) for entry in ordered_roles(roles_dir, group)]]
|
||||
except Exception as exc:
|
||||
raise AnsibleError(f"group_roles: {exc}") from exc
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Lookup `stage_groups`: the ordered role-group names
|
||||
(``tasks/groups/<g>-roles.yml``) that belong to a deploy stage, in intra-stage
|
||||
"""Lookup `stage_groups`: the ordered role-group names that belong to a
|
||||
deploy stage, in intra-stage
|
||||
call order. The stage plays loop over it so group membership follows
|
||||
``meta/categories.yml`` instead of a hardcoded list:
|
||||
|
||||
|
||||
@@ -12,20 +12,6 @@ fi
|
||||
|
||||
ROLES_DIR="./roles"
|
||||
|
||||
INCLUDES_SCRIPT="./cli/build/include/__main__.py"
|
||||
INCLUDES_OUT_DIR="./tasks/groups"
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Validation
|
||||
# ------------------------------------------------------------
|
||||
require_file() {
|
||||
local path="$1"
|
||||
[[ -f "$path" ]] || {
|
||||
echo "❌ File not found: $path" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
require_dir() {
|
||||
local path="$1"
|
||||
[[ -d "$path" ]] || {
|
||||
@@ -43,42 +29,6 @@ require_cmd() {
|
||||
|
||||
require_cmd "${PYTHON}"
|
||||
require_dir "${ROLES_DIR}"
|
||||
require_file "${INCLUDES_SCRIPT}"
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------
|
||||
log_section() {
|
||||
echo
|
||||
echo "------------------------------------------------------------"
|
||||
echo "$1"
|
||||
echo "------------------------------------------------------------"
|
||||
}
|
||||
|
||||
log_section "🧩 Generating role include files"
|
||||
mkdir -p "${INCLUDES_OUT_DIR}"
|
||||
|
||||
mapfile -t INCLUDE_GROUPS < <("${PYTHON}" -m cli.meta.categories.invokable -s "-")
|
||||
|
||||
for existing in "${INCLUDES_OUT_DIR}"/*-roles.yml; do
|
||||
[[ -e "${existing}" ]] || continue
|
||||
keep=0
|
||||
for grp in "${INCLUDE_GROUPS[@]}"; do
|
||||
[[ "${existing}" == "${INCLUDES_OUT_DIR}/${grp}roles.yml" ]] && keep=1
|
||||
done
|
||||
if [[ "${keep}" -eq 0 ]]; then
|
||||
echo "→ Removing stale ${existing}"
|
||||
rm -f "${existing}"
|
||||
fi
|
||||
done
|
||||
|
||||
for grp in "${INCLUDE_GROUPS[@]}"; do
|
||||
[[ -z "${grp}" ]] && continue
|
||||
out="${INCLUDES_OUT_DIR}/${grp}roles.yml"
|
||||
echo "→ Building ${out} (pattern: '${grp}')"
|
||||
"${PYTHON}" "${INCLUDES_SCRIPT}" "${ROLES_DIR}" -p "${grp}" -o "${out}"
|
||||
echo " ✅ ${out}"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "🎉 Project setup completed"
|
||||
+2
-8
@@ -1,9 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
# Remove the container-owned generated artefacts under the working tree.
|
||||
#
|
||||
# Both `build/` (install-stamp) and `tasks/groups/*-roles.yml` (output
|
||||
# of `cli build include` during setup) are produced inside the compose
|
||||
# container by the in-container UID (typically `nobody`). The host
|
||||
# `build/` (install-stamp) is produced inside the compose container by
|
||||
# the in-container UID (typically `nobody`). The host
|
||||
# user cannot rm them directly. This helper starts the infinito
|
||||
# compose container if a stopped one exists, then deletes the
|
||||
# nobody-owned artefacts from inside it. Safe no-op when the targets
|
||||
@@ -30,15 +29,10 @@ if ! docker ps --filter "name=${INFINITO_CONTAINER}" --format '{{.Names}}' | gre
|
||||
fi
|
||||
fi
|
||||
|
||||
# The tasks/groups/*.yml files are regenerated by `make setup`; the
|
||||
# build/install.stamp is regenerated by `make install`. Both are safe
|
||||
# to delete unconditionally — the next make run will produce them
|
||||
# fresh with whatever UID is then current.
|
||||
echo "[clean-container-owned] removing container-owned generated artefacts inside ${INFINITO_CONTAINER}"
|
||||
docker exec -i "${INFINITO_CONTAINER}" sh -lc "
|
||||
set -eu
|
||||
cd '${INFINITO_SRC_DIR}'
|
||||
rm -rf build
|
||||
find tasks/groups -mindepth 1 -name '*-roles.yml' -delete
|
||||
"
|
||||
echo "[clean-container-owned] done"
|
||||
@@ -1 +0,0 @@
|
||||
*-roles.yml
|
||||
@@ -20,8 +20,6 @@
|
||||
failed_when: dns_check.rc != 0
|
||||
when:
|
||||
- MODE_ASSERT | bool
|
||||
# .onion domains are Tor-only and intentionally not DNS-resolvable, so this
|
||||
# clearnet sanity check does not apply to an onion node.
|
||||
- not (DOMAIN_PRIMARY is search('\.onion$'))
|
||||
|
||||
- name: 🔀 Merge variables
|
||||
@@ -168,13 +166,13 @@
|
||||
changed_when: false
|
||||
|
||||
- name: "🥾 Bootstrap NFS clients + registry (MUST precede Preload)"
|
||||
ansible.builtin.include_tasks: "./tasks/groups/{{ role_group }}-roles.yml"
|
||||
ansible.builtin.include_tasks: ./tasks/utils/setup/group.yml
|
||||
loop:
|
||||
- svc-storage
|
||||
- svc-registry
|
||||
loop_control:
|
||||
loop_var: role_group
|
||||
label: "{{ role_group }}-roles.yml"
|
||||
label: "{{ role_group }}"
|
||||
|
||||
- name: 📂 Include Token Store
|
||||
ansible.builtin.include_role:
|
||||
@@ -186,11 +184,11 @@
|
||||
name: sys-service-loader
|
||||
|
||||
- name: "🧩 Load base roles"
|
||||
ansible.builtin.include_tasks: "./tasks/groups/{{ role_group }}-roles.yml"
|
||||
ansible.builtin.include_tasks: ./tasks/utils/setup/group.yml
|
||||
loop: "{{ lookup('stage_groups', 'constructor') }}"
|
||||
loop_control:
|
||||
loop_var: role_group
|
||||
label: "{{ role_group }}-roles.yml"
|
||||
label: "{{ role_group }}"
|
||||
|
||||
- name: 📦 Install inventory-defined packages
|
||||
package_install:
|
||||
|
||||
@@ -23,11 +23,11 @@
|
||||
when: "'svc-swarm-node' in group_names"
|
||||
|
||||
- name: "📂 Include server roles"
|
||||
include_tasks: "./tasks/groups/{{ server_group }}-roles.yml"
|
||||
include_tasks: ./tasks/utils/setup/group.yml
|
||||
loop: "{{ lookup('stage_groups', 'server') }}"
|
||||
loop_control:
|
||||
loop_var: server_group
|
||||
label: "{{ server_group }}-roles.yml"
|
||||
loop_var: role_group
|
||||
label: "{{ role_group }}"
|
||||
|
||||
- name: ▶️ Run E2E Playwright tests
|
||||
include_role:
|
||||
|
||||
@@ -4,11 +4,11 @@
|
||||
name: "user-workstation"
|
||||
|
||||
- name: "🔧 setup docker role includes for workstations"
|
||||
include_tasks: "./tasks/groups/{{ workstation_group }}-roles.yml"
|
||||
include_tasks: ./tasks/utils/setup/group.yml
|
||||
loop: "{{ lookup('stage_groups', 'workstation') }}"
|
||||
loop_control:
|
||||
loop_var: workstation_group
|
||||
label: "{{ workstation_group }}-roles.yml"
|
||||
loop_var: role_group
|
||||
label: "{{ role_group }}"
|
||||
|
||||
- name: 🖥️ general host setup
|
||||
include_role:
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
---
|
||||
- name: "🧩 Load destruction roles"
|
||||
include_tasks: "./tasks/groups/{{ destruction_group }}-roles.yml"
|
||||
include_tasks: ./tasks/utils/setup/group.yml
|
||||
loop: "{{ lookup('stage_groups', 'destructor') }}"
|
||||
loop_control:
|
||||
loop_var: destruction_group
|
||||
label: "{{ destruction_group }}-roles.yml"
|
||||
loop_var: role_group
|
||||
label: "{{ role_group }}"
|
||||
|
||||
- name: "🛠️ Finalize services"
|
||||
include_role:
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
---
|
||||
- name: "🧩 setup {{ role_group }} roles in run_after order"
|
||||
ansible.builtin.include_tasks: ./tasks/utils/setup/role.yml
|
||||
when: (role_entry.app | application_allowed(group_names, lookup('deployment').whitelist))
|
||||
loop: "{{ lookup('group_roles', role_group) }}"
|
||||
loop_control:
|
||||
loop_var: role_entry
|
||||
label: "{{ role_entry.app }}"
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
- name: "🔧 setup {{ role_entry.app }}"
|
||||
ansible.builtin.include_role:
|
||||
name: "{{ role_entry.role }}"
|
||||
|
||||
- name: "🚿 flush handlers after {{ role_entry.app }}"
|
||||
ansible.builtin.meta: flush_handlers
|
||||
@@ -1,6 +1,6 @@
|
||||
# Playbook Tests 🎬
|
||||
|
||||
Integration tests for playbook-level wiring: `tasks/stages/**`, `tasks/groups/*-roles.yml`, invokable-path coverage, and the dispatch between them.
|
||||
Integration tests for playbook-level wiring: `tasks/stages/**`, invokable-path coverage, and the dispatch between them.
|
||||
|
||||
Tests in this directory MUST only cover rules about how playbooks invoke role groups and how the invokable-paths filter maps to stages. Tests for role-internal `meta/main.yml` MUST live under [`roles/meta/`](../../roles/meta/); tests for include/import target existence MUST live under [`roles/includes/`](../../roles/includes/).
|
||||
|
||||
|
||||
@@ -12,13 +12,11 @@ ROOT = PROJECT_ROOT
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
STAGES_DIR = ROOT / "tasks" / "stages"
|
||||
GROUPS_DIR = ROOT / "tasks" / "groups"
|
||||
|
||||
|
||||
class TestMetaRolesIntegration(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.role_files = get_invokable_paths(suffix="-roles.yml")
|
||||
cls.invokable_items = get_invokable_paths()
|
||||
|
||||
cls.playbook_contents = {}
|
||||
@@ -26,7 +24,7 @@ class TestMetaRolesIntegration(unittest.TestCase):
|
||||
cls.playbook_contents[path] = read_text(str(path))
|
||||
|
||||
cls.include_pattern = re.compile(
|
||||
r'include_tasks:\s*["\']\./tasks/groups/\{\{\s*[A-Za-z_][A-Za-z0-9_]*\s*\}\}-roles\.yml["\']'
|
||||
r'include_tasks:\s*["\']?\./tasks/utils/setup/group\.yml["\']?'
|
||||
)
|
||||
|
||||
stage_lookup = re.compile(
|
||||
@@ -37,39 +35,20 @@ class TestMetaRolesIntegration(unittest.TestCase):
|
||||
for match in stage_lookup.finditer(content):
|
||||
cls.lookup_referenced.update(stage_groups(match.group("stage")))
|
||||
|
||||
def test_all_role_files_exist(self):
|
||||
"""Each '-roles.yml' path returned by the filter must exist in the project root."""
|
||||
missing = []
|
||||
for fname in self.role_files:
|
||||
path = GROUPS_DIR / fname
|
||||
if not path.is_file():
|
||||
missing.append(fname)
|
||||
self.assertFalse(
|
||||
missing, f"The following role files are missing at project root: {missing}"
|
||||
)
|
||||
|
||||
def test_each_invokable_item_referenced_in_playbooks(self):
|
||||
"""
|
||||
Each invokable item (without suffix) must be either:
|
||||
Each invokable item must be either:
|
||||
- resolved by a `lookup('stage_groups', '<stage>')` loop, or
|
||||
- looped through by a dynamic include_tasks ({{ item }}-roles.yml), or
|
||||
- referenced by a direct include_tasks to ./tasks/groups/<item>-roles.yml.
|
||||
- named in the loop of a stage that includes ./tasks/utils/setup/group.yml.
|
||||
"""
|
||||
not_referenced = []
|
||||
for item in self.invokable_items:
|
||||
found = item in self.lookup_referenced
|
||||
loop_entry = re.compile(rf"-\s*{re.escape(item)}\b")
|
||||
direct_include = re.compile(
|
||||
rf'include_tasks:\s*["\']\./tasks/groups/{re.escape(item)}-roles\.yml["\']'
|
||||
)
|
||||
for content in self.playbook_contents.values():
|
||||
if found:
|
||||
break
|
||||
dynamic_ref = self.include_pattern.search(
|
||||
content
|
||||
) and loop_entry.search(content)
|
||||
static_ref = direct_include.search(content)
|
||||
if dynamic_ref or static_ref:
|
||||
if self.include_pattern.search(content) and loop_entry.search(content):
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import unittest
|
||||
|
||||
# import the functions from your CLI script
|
||||
from cli.build.include import build_dependency_graph, find_cycle
|
||||
from utils.roles.order import build_dependency_graph, find_cycle
|
||||
|
||||
|
||||
class TestCircularDependencies(unittest.TestCase):
|
||||
@@ -17,18 +16,14 @@ class TestCircularDependencies(unittest.TestCase):
|
||||
cls.roles_dir = str(PROJECT_ROOT / "roles")
|
||||
|
||||
def test_no_circular_dependencies(self):
|
||||
# Build the dependency graph using the real roles/
|
||||
_graph, _in_degree, roles = build_dependency_graph(self.roles_dir)
|
||||
|
||||
# Attempt to find a cycle in the run_after mapping
|
||||
cycle = find_cycle(roles)
|
||||
|
||||
if cycle:
|
||||
# Format cycle as "A -> B -> C -> A"
|
||||
cycle_str = " -> ".join(cycle)
|
||||
self.fail(f"Circular dependency detected among roles: {cycle_str}")
|
||||
|
||||
# If no cycle, this assertion will pass
|
||||
self.assertIsNone(cycle, "Expected no circular dependencies")
|
||||
|
||||
|
||||
|
||||
@@ -26,7 +26,6 @@ from . import PROJECT_ROOT
|
||||
|
||||
MAX_ITEMS_PER_FOLDER = 12
|
||||
|
||||
# Flat structure folders that intentionally stay one level deep.
|
||||
FLAT_STRUCTURE_WHITELIST = {
|
||||
".github/workflows",
|
||||
"cli",
|
||||
@@ -38,7 +37,6 @@ FLAT_STRUCTURE_WHITELIST = {
|
||||
"roles",
|
||||
"inventories/bundles/servers",
|
||||
"inventories/bundles/workstations",
|
||||
"tasks/groups",
|
||||
"tests/unit/plugins/filter",
|
||||
"tests/unit/roles",
|
||||
"tests/unit/plugins/lookup",
|
||||
@@ -67,8 +65,6 @@ def _tracked_paths(root: Path) -> list[Path]:
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
except Exception:
|
||||
# `iter_project_files` already prunes `.git`, `__pycache__`,
|
||||
# `.cache`, `.venv`, `node_modules`, etc.
|
||||
return [Path(p) for p in iter_project_files()]
|
||||
|
||||
rel_paths = [
|
||||
@@ -112,7 +108,6 @@ def _is_whitelisted_folder(folder: Path) -> bool:
|
||||
rel = folder.as_posix()
|
||||
if rel in FLAT_STRUCTURE_WHITELIST:
|
||||
return True
|
||||
# The first level below roles/ is intentionally flat by design.
|
||||
return len(folder.parts) == 2 and folder.parts[0] == "roles"
|
||||
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import unittest
|
||||
import tempfile
|
||||
import shutil
|
||||
from cli.build.include import build_dependency_graph, topological_sort, gen_condi_role_incl
|
||||
|
||||
from utils.cache.yaml import dump_yaml
|
||||
from utils.roles.mapping import ROLE_FILE_META_MAIN, ROLE_FILE_META_SERVICES, ROLE_FILE_VARS_MAIN
|
||||
|
||||
class TestGeneratePlaybook(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# Create a temporary directory to simulate roles
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
|
||||
# Define mock roles and dependencies
|
||||
self.roles = {
|
||||
'role-a': {'run_after': [], 'application_id': 'a'},
|
||||
'role-b': {'run_after': ['role-a'], 'application_id': 'b'},
|
||||
'role-c': {'run_after': ['role-b'], 'application_id': 'c'},
|
||||
'role-d': {'run_after': [], 'application_id': 'd'},
|
||||
}
|
||||
|
||||
for role_name, meta in self.roles.items():
|
||||
role_path = os.path.join(self.temp_dir, role_name)
|
||||
os.makedirs(os.path.join(role_path, 'meta'), exist_ok=True)
|
||||
os.makedirs(os.path.join(role_path, 'vars'), exist_ok=True)
|
||||
|
||||
# In the new layout run_after lives at
|
||||
# meta/services.yml.<primary_entity>.run_after. For role names
|
||||
# that are not prefixed by a known category (e.g. 'role-a'),
|
||||
# get_entity_name returns the role name itself.
|
||||
meta_services = {
|
||||
role_name: {
|
||||
'run_after': meta['run_after'],
|
||||
}
|
||||
}
|
||||
|
||||
vars_file = {
|
||||
'application_id': meta['application_id']
|
||||
}
|
||||
|
||||
dump_yaml(os.path.join(role_path, ROLE_FILE_META_SERVICES), meta_services)
|
||||
|
||||
# find_roles() still uses meta/main.yml as a marker file to
|
||||
# detect roles, so we create an empty one here.
|
||||
dump_yaml(os.path.join(role_path, ROLE_FILE_META_MAIN), {})
|
||||
|
||||
dump_yaml(os.path.join(role_path, ROLE_FILE_VARS_MAIN), vars_file)
|
||||
|
||||
def tearDown(self):
|
||||
# Clean up the temporary directory
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_dependency_graph_and_sort(self):
|
||||
graph, in_degree, roles = build_dependency_graph(self.temp_dir)
|
||||
|
||||
self.assertIn('role-a', graph)
|
||||
self.assertIn('role-b', graph)
|
||||
self.assertEqual(graph['role-a'], ['role-b'])
|
||||
self.assertEqual(graph['role-b'], ['role-c'])
|
||||
self.assertEqual(graph['role-c'], [])
|
||||
self.assertEqual(in_degree['role-c'], 1)
|
||||
self.assertEqual(in_degree['role-b'], 1)
|
||||
self.assertEqual(in_degree['role-a'], 0)
|
||||
self.assertEqual(in_degree['role-d'], 0)
|
||||
|
||||
sorted_roles = topological_sort(graph, in_degree)
|
||||
# The expected order must be a → b → c, d can be anywhere before or after
|
||||
self.assertTrue(sorted_roles.index('role-a') < sorted_roles.index('role-b') < sorted_roles.index('role-c'))
|
||||
|
||||
def test_gen_condi_role_incl(self):
|
||||
entries = gen_condi_role_incl(self.temp_dir)
|
||||
|
||||
text = ''.join(entries)
|
||||
self.assertIn("setup a", text)
|
||||
self.assertIn("setup b", text)
|
||||
self.assertIn("setup c", text)
|
||||
self.assertIn("setup d", text)
|
||||
|
||||
# Order must preserve run_after
|
||||
a_index = text.index("setup a")
|
||||
b_index = text.index("setup b")
|
||||
c_index = text.index("setup c")
|
||||
self.assertTrue(a_index < b_index < c_index)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,86 @@
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from utils.cache.yaml import dump_yaml
|
||||
from utils.roles.mapping import (
|
||||
ROLE_FILE_META_MAIN,
|
||||
ROLE_FILE_META_SERVICES,
|
||||
ROLE_FILE_VARS_MAIN,
|
||||
)
|
||||
from utils.roles.order import (
|
||||
build_dependency_graph,
|
||||
find_cycle,
|
||||
ordered_roles,
|
||||
topological_sort,
|
||||
)
|
||||
|
||||
|
||||
class TestRoleOrder(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.roles = {
|
||||
"grp-a": {"run_after": [], "application_id": "a"},
|
||||
"grp-b": {"run_after": ["grp-a"], "application_id": "b"},
|
||||
"grp-c": {"run_after": ["grp-b"], "application_id": "c"},
|
||||
"grp-d": {"run_after": [], "application_id": "d"},
|
||||
}
|
||||
|
||||
for role_name, meta in self.roles.items():
|
||||
role_path = Path(self.temp_dir) / role_name
|
||||
(role_path / "meta").mkdir(parents=True, exist_ok=True)
|
||||
(role_path / "vars").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
dump_yaml(
|
||||
str(role_path / ROLE_FILE_META_SERVICES),
|
||||
{role_name: {"run_after": meta["run_after"]}},
|
||||
)
|
||||
dump_yaml(str(role_path / ROLE_FILE_META_MAIN), {})
|
||||
dump_yaml(
|
||||
str(role_path / ROLE_FILE_VARS_MAIN),
|
||||
{"application_id": meta["application_id"]},
|
||||
)
|
||||
|
||||
def tearDown(self):
|
||||
ordered_roles.cache_clear()
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_dependency_graph_and_sort(self):
|
||||
graph, in_degree, _roles = build_dependency_graph(self.temp_dir)
|
||||
|
||||
self.assertEqual(graph["grp-a"], ["grp-b"])
|
||||
self.assertEqual(graph["grp-b"], ["grp-c"])
|
||||
self.assertEqual(in_degree["grp-a"], 0)
|
||||
self.assertEqual(in_degree["grp-d"], 0)
|
||||
|
||||
order = topological_sort(graph, in_degree)
|
||||
self.assertTrue(
|
||||
order.index("grp-a") < order.index("grp-b") < order.index("grp-c")
|
||||
)
|
||||
|
||||
def test_no_cycle_in_a_linear_chain(self):
|
||||
_graph, _in_degree, roles = build_dependency_graph(self.temp_dir)
|
||||
self.assertIsNone(find_cycle(roles))
|
||||
|
||||
def test_ordered_roles_pairs_role_with_application_id(self):
|
||||
entries = ordered_roles(self.temp_dir, "grp")
|
||||
|
||||
self.assertEqual(sorted(e["app"] for e in entries), ["a", "b", "c", "d"])
|
||||
for entry in entries:
|
||||
self.assertEqual(entry["role"], f"grp-{entry['app']}")
|
||||
|
||||
apps = [e["app"] for e in entries]
|
||||
self.assertTrue(
|
||||
apps.index("a") < apps.index("b") < apps.index("c"),
|
||||
"the group must be returned in run_after order",
|
||||
)
|
||||
|
||||
def test_missing_application_id_is_rejected(self):
|
||||
dump_yaml(str(Path(self.temp_dir) / "grp-d" / ROLE_FILE_VARS_MAIN), {})
|
||||
with self.assertRaises(ValueError):
|
||||
ordered_roles(self.temp_dir, "grp")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,189 @@
|
||||
"""Resolve the roles of a group into their run_after order.
|
||||
|
||||
The play used to consume a task file per group, generated at setup time. The
|
||||
order is derived from data that already lives in the roles, so it is resolved
|
||||
here instead and read through the `group_roles` lookup.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict, deque
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from utils.cache.yaml import load_yaml
|
||||
from utils.roles.mapping import ROLE_FILE_META_MAIN, ROLE_FILE_VARS_MAIN
|
||||
|
||||
|
||||
def find_roles(roles_dir: str | Path, prefixes: list[str] | None = None):
|
||||
"""Yield ``(role_path, meta_file)`` for every role matching *prefixes*.
|
||||
|
||||
Args:
|
||||
roles_dir: directory holding the role folders.
|
||||
prefixes: name prefixes to keep; all roles when empty or None.
|
||||
"""
|
||||
for path in sorted(Path(roles_dir).iterdir()):
|
||||
if prefixes and not any(
|
||||
path.name.startswith(pref) or path.name == pref.rstrip("-")
|
||||
for pref in prefixes
|
||||
):
|
||||
continue
|
||||
meta_file = path / ROLE_FILE_META_MAIN
|
||||
if path.is_dir() and meta_file.is_file():
|
||||
yield path, meta_file
|
||||
|
||||
|
||||
def load_run_after(meta_file: str | Path) -> list[str]:
|
||||
"""Return the run_after list declared by the role owning *meta_file*."""
|
||||
from utils.roles.meta_lookup import get_role_run_after
|
||||
|
||||
role_path = Path(meta_file).parent.parent.resolve()
|
||||
try:
|
||||
return get_role_run_after(str(role_path), role_name=role_path.name)
|
||||
except Exception: # noqa: BLE001 a role without readable run_after simply has none
|
||||
return []
|
||||
|
||||
|
||||
def load_application_id(role_path: str | Path) -> str | None:
|
||||
"""Return the application_id from the role's vars/main.yml, or None."""
|
||||
vars_file = Path(role_path) / ROLE_FILE_VARS_MAIN
|
||||
if vars_file.exists():
|
||||
return load_yaml(str(vars_file)).get("application_id")
|
||||
return None
|
||||
|
||||
|
||||
def build_dependency_graph(roles_dir: str | Path, prefixes: list[str] | None = None):
|
||||
"""Return ``(graph, in_degree, roles)`` over the roles matching *prefixes*.
|
||||
|
||||
Only run_after edges pointing inside the selected set become edges, so a
|
||||
group resolves independently of the roles outside it.
|
||||
"""
|
||||
graph: dict[str, list[str]] = defaultdict(list)
|
||||
in_degree: dict[str, int] = defaultdict(int)
|
||||
roles: dict[str, dict[str, Any]] = {}
|
||||
|
||||
role_entries = list(find_roles(roles_dir, prefixes))
|
||||
in_scope = {path.name for path, _ in role_entries}
|
||||
|
||||
for role_path, meta_file in role_entries:
|
||||
role_name = role_path.name
|
||||
in_scope_deps = [d for d in load_run_after(meta_file) if d in in_scope]
|
||||
|
||||
roles[role_name] = {
|
||||
"role_name": role_name,
|
||||
"run_after": in_scope_deps,
|
||||
"application_id": load_application_id(role_path),
|
||||
"path": str(role_path),
|
||||
}
|
||||
|
||||
for dependency in in_scope_deps:
|
||||
graph[dependency].append(role_name)
|
||||
in_degree[role_name] += 1
|
||||
|
||||
if role_name not in in_degree:
|
||||
in_degree[role_name] = 0
|
||||
|
||||
return graph, in_degree, roles
|
||||
|
||||
|
||||
def find_cycle(roles: dict[str, dict[str, Any]]) -> list[str] | None:
|
||||
"""Return a run_after cycle as a role list with the start repeated, or None."""
|
||||
visited: set[str] = set()
|
||||
stack: set[str] = set()
|
||||
|
||||
def dfs(node: str, path: list[str]) -> list[str] | None:
|
||||
visited.add(node)
|
||||
stack.add(node)
|
||||
path.append(node)
|
||||
for dep in roles.get(node, {}).get("run_after", []):
|
||||
if dep not in visited:
|
||||
found = dfs(dep, path)
|
||||
if found:
|
||||
return found
|
||||
elif dep in stack:
|
||||
return [*path[path.index(dep) :], dep]
|
||||
stack.remove(node)
|
||||
path.pop()
|
||||
return None
|
||||
|
||||
for role in roles:
|
||||
if role not in visited:
|
||||
cycle = dfs(role, [])
|
||||
if cycle:
|
||||
return cycle
|
||||
return None
|
||||
|
||||
|
||||
def topological_sort(graph, in_degree, roles=None) -> list[str]:
|
||||
"""Return the role names in run_after order.
|
||||
|
||||
Raises:
|
||||
RuntimeError: when the graph does not resolve, naming the cycle and the
|
||||
roles left unsorted so the failure is actionable.
|
||||
"""
|
||||
queue = deque([r for r, d in in_degree.items() if d == 0])
|
||||
sorted_roles: list[str] = []
|
||||
local_in = dict(in_degree)
|
||||
|
||||
while queue:
|
||||
role = queue.popleft()
|
||||
sorted_roles.append(role)
|
||||
for nbr in graph.get(role, []):
|
||||
local_in[nbr] -= 1
|
||||
if local_in[nbr] == 0:
|
||||
queue.append(nbr)
|
||||
|
||||
if len(sorted_roles) != len(in_degree):
|
||||
cycle = find_cycle(roles or {})
|
||||
unsorted = [r for r in in_degree if r not in sorted_roles]
|
||||
|
||||
reason = (
|
||||
f"Circular dependency detected: {' -> '.join(cycle)}"
|
||||
if cycle
|
||||
else "Unresolved dependencies among roles (possible cycle or missing role)."
|
||||
)
|
||||
details = []
|
||||
if unsorted:
|
||||
details.append("Unsorted roles and their declared run_after dependencies:")
|
||||
details += [
|
||||
f" - {r} depends on {roles.get(r, {}).get('run_after', [])!r}"
|
||||
for r in unsorted
|
||||
]
|
||||
|
||||
raise RuntimeError(
|
||||
"\n".join(
|
||||
[
|
||||
"❌ Dependency resolution failed",
|
||||
reason,
|
||||
*details,
|
||||
f"Full dependency graph: {dict(graph)!r}",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return sorted_roles
|
||||
|
||||
|
||||
@cache
|
||||
def ordered_roles(roles_dir: str, group: str) -> tuple[dict[str, str], ...]:
|
||||
"""Return ``({'role': …, 'app': …}, …)`` for *group*, in run_after order.
|
||||
|
||||
Args:
|
||||
roles_dir: directory holding the role folders.
|
||||
group: group name without the trailing dash, e.g. ``web-app``.
|
||||
|
||||
Raises:
|
||||
ValueError: when a role in the group declares no application_id.
|
||||
"""
|
||||
graph, in_degree, roles = build_dependency_graph(roles_dir, [f"{group}-"])
|
||||
|
||||
entries = []
|
||||
for role_name in topological_sort(graph, in_degree, roles):
|
||||
role = roles[role_name]
|
||||
if role.get("application_id") is None:
|
||||
vars_file = Path(role["path"]) / ROLE_FILE_VARS_MAIN
|
||||
raise ValueError(f"'application_id' missing in {vars_file}")
|
||||
entries.append({"role": role_name, "app": role["application_id"]})
|
||||
|
||||
return tuple(entries)
|
||||
@@ -9,7 +9,6 @@ and the ``stage_order`` / ``role_stage`` lookups read one source of truth.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from utils import PROJECT_ROOT
|
||||
from utils.cache.yaml import load_yaml
|
||||
from utils.roles.applications.services.registry import run_after_topological_order
|
||||
from utils.roles.categories import categories_file
|
||||
@@ -102,10 +101,10 @@ def role_sort_key(role: str) -> tuple:
|
||||
|
||||
|
||||
def _group_names() -> list[str]:
|
||||
groups_dir = PROJECT_ROOT / "tasks" / "groups"
|
||||
if not groups_dir.is_dir():
|
||||
return []
|
||||
return [p.name.removesuffix("-roles.yml") for p in groups_dir.glob("*-roles.yml")]
|
||||
"""The invokable category paths — the role groups a stage can loop over."""
|
||||
from plugins.filter.invokable_paths import get_invokable_paths
|
||||
|
||||
return get_invokable_paths()
|
||||
|
||||
|
||||
def _bootstrap_prefixes() -> set[str]:
|
||||
@@ -117,7 +116,7 @@ def _bootstrap_prefixes() -> set[str]:
|
||||
|
||||
|
||||
def stage_groups(stage: str) -> list[str]:
|
||||
"""Ordered role-group names (``tasks/groups/<g>-roles.yml``) that belong to
|
||||
"""Ordered role-group names that belong to
|
||||
``stage``, in intra-stage call order (category run_after, then name). The
|
||||
SPOT the stage plays consume so group membership follows categories.yml.
|
||||
Bootstrap-flagged categories are excluded (they run via explicit steps)."""
|
||||
|
||||
Reference in new issue
Block a user