From 634f1835fc97845012b266bc90adfc66956b12cf Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 7 Jul 2025 04:31:43 +0200 Subject: [PATCH] Optimized role creation script --- cli/create_docker_role.py | 175 ++++++++++++------ templates/docker_role/meta/main.yml.j2 | 24 +-- templates/docker_role/tasks/main.yml.j2 | 2 +- .../templates/docker-compose.yml.j2.j2 | 3 +- .../docker_role/vars/configuration.yml.j2 | 4 + templates/docker_role/vars/main.yml.j2 | 2 +- 6 files changed, 131 insertions(+), 79 deletions(-) diff --git a/cli/create_docker_role.py b/cli/create_docker_role.py index 6adef5f7..4fdaa3e8 100644 --- a/cli/create_docker_role.py +++ b/cli/create_docker_role.py @@ -2,14 +2,16 @@ import argparse import os import shutil +import sys import yaml import ipaddress +import difflib from jinja2 import Environment, FileSystemLoader # Paths to the group-vars files PORTS_FILE = './group_vars/all/09_ports.yml' NETWORKS_FILE = './group_vars/all/10_networks.yml' -ROLE_TEMPLATE_DIR = './docker-template' +ROLE_TEMPLATE_DIR = './templates/docker_role' ROLES_DIR = './roles' @@ -24,35 +26,44 @@ def dump_yaml(data, path): def get_next_network(networks_dict, prefixlen): - # Collect all local subnets matching the given prefix length + """Select the first available local subnet with the given prefix length.""" nets = [] - for name, info in networks_dict['defaults_networks']['local'].items(): + for info in networks_dict['defaults_networks']['local'].values(): net = ipaddress.ip_network(info['subnet']) if net.prefixlen == prefixlen: nets.append(net) - # Sort by network address and return the first one nets.sort(key=lambda n: int(n.network_address)) return nets[0] -def get_next_port(ports_dict, category, service): - used = set() - # Gather already taken ports under localhost.category - for svc, port in ports_dict['ports']['localhost'].get(category, {}).items(): - used.add(int(port)) - # Start searching from port 1 upwards +def get_next_port(ports_dict, category): + """Find the next unused port in the given localhost category.""" + used = {int(p) for p in ports_dict['ports']['localhost'].get(category, {}).values()} candidate = 1 while candidate in used: candidate += 1 return candidate +def prompt_conflict(dst_file): + """Prompt the user to resolve a file conflict.""" + print(f"Conflict detected: {dst_file}") + print("Choose action: [1] overwrite, [2] skip, [3] merge") + choice = None + while choice not in ('1', '2', '3'): + choice = input("Enter 1, 2, or 3: ").strip() + return choice + + def render_template(src_dir, dst_dir, context): + """Recursively render all templates from src_dir into dst_dir, handling conflicts.""" env = Environment( loader=FileSystemLoader(src_dir), keep_trailing_newline=True, autoescape=False, ) + # Add a bool filter for Jinja evaluation + env.filters['bool'] = lambda x: bool(x) for root, _, files in os.walk(src_dir): rel_path = os.path.relpath(root, src_dir) target_path = os.path.join(dst_dir, rel_path) @@ -61,83 +72,127 @@ def render_template(src_dir, dst_dir, context): template = env.get_template(os.path.join(rel_path, filename)) rendered = template.render(**context) out_name = filename[:-3] if filename.endswith('.j2') else filename - with open(os.path.join(target_path, out_name), 'w') as f: + dst_file = os.path.join(target_path, out_name) + + if os.path.exists(dst_file): + choice = prompt_conflict(dst_file) + if choice == '2': + print(f"Skipping {dst_file}") + continue + if choice == '3': + with open(dst_file) as f_old: + old_lines = f_old.readlines() + new_lines = rendered.splitlines(keepends=True) + diff = difflib.unified_diff( + old_lines, new_lines, + fromfile=f"a/{out_name}", + tofile=f"b/{out_name}", + lineterm='' + ) + diff_path = dst_file + '.diff' + with open(diff_path, 'w') as fd: + fd.writelines(diff) + print(f"Diff written to {diff_path}; please merge manually.") + continue + # Overwrite + print(f"Overwriting {dst_file}") + + with open(dst_file, 'w') as f: f.write(rendered) def main(): + # Load current port categories dynamically + ports_yaml = load_yaml(PORTS_FILE) + categories = list(ports_yaml.get('ports', {}).get('localhost', {}).keys()) + parser = argparse.ArgumentParser( - description="Create a Docker Ansible role with Jinja2 templates, and assign network and ports" + description="Create or update a Docker Ansible role, and assign network and ports" ) parser.add_argument( - '--application-id', '-a', required=True, - help="Unique ID of the application (used in the role name)" + '-a', '--application-id', required=True, + help="Unique application ID" ) parser.add_argument( - '--network', '-n', choices=['24', '28'], required=True, - help="Network prefix length to assign (/24 or /28)" + '-n', '--network', choices=['24', '28'], required=True, + help="Network prefix length (/24 or /28)" ) parser.add_argument( - '--ports', '-p', nargs='+', metavar="CATEGORY.SERVICE", required=True, - help="List of ports in the format category.service (e.g. http.nextcloud)" + '-p', '--ports', nargs='+', choices=categories, required=True, + help=f"Port categories to assign (allowed: {', '.join(categories)})" ) args = parser.parse_args() app_id = args.application_id role_name = f"docker-{app_id}" - - # 1) Create the role from the template role_dir = os.path.join(ROLES_DIR, role_name) + + # If role directory exists, ask whether to continue if os.path.exists(role_dir): - parser.error(f"Role {role_name} already exists at {role_dir}") + cont = input(f"Role {role_name} already exists. Continue updating? [y/N]: ").strip().lower() + if cont != 'y': + print("Aborting.") + sys.exit(1) + else: + os.makedirs(role_dir) + + # 1) Render and copy templates, with conflict resolution + # Provide database_type=0 in context for task template logic render_template(ROLE_TEMPLATE_DIR, role_dir, { 'application_id': app_id, 'role_name': role_name, + 'database_type': 0, }) - print(f"→ Role {role_name} created at {role_dir}") + print(f"→ Templates rendered into {role_dir}") - # 2) Assign network - networks = load_yaml(NETWORKS_FILE) - prefix = int(args.network) - chosen_net = get_next_network(networks, prefix) - out_net = { - 'defaults_networks': { - 'application': { - app_id: str(chosen_net) - } - } - } - net_file = f'./group_vars/{app_id}_network.yml' - dump_yaml(out_net, net_file) - print(f"→ Assigned network {chosen_net} (/{prefix}) and wrote to {net_file}") + # 2) Assign network if not already set + net_vars_file = f'./group_vars/{app_id}_network.yml' + if os.path.exists(net_vars_file): + existing_net = load_yaml(net_vars_file) + apps = existing_net.get('defaults_networks', {}).get('application', {}) + if app_id in apps: + print(f"→ Network for {app_id} already configured ({apps[app_id]}), skipping.") + else: + networks = load_yaml(NETWORKS_FILE) + net = get_next_network(networks, int(args.network)) + apps[app_id] = str(net) + dump_yaml(existing_net, net_vars_file) + print(f"→ Appended network {net} for {app_id}") + else: + networks = load_yaml(NETWORKS_FILE) + net = get_next_network(networks, int(args.network)) + dump_yaml({'defaults_networks': {'application': {app_id: str(net)}}}, net_vars_file) + print(f"→ Created network vars file with {net} for {app_id}") - # 3) Assign ports - ports_yaml = load_yaml(PORTS_FILE) + # 3) Assign ports if not present assigned = {} - for entry in args.ports: - try: - category, service = entry.split('.', 1) - except ValueError: - parser.error(f"Invalid port spec: {entry}. Must be CATEGORY.SERVICE") - port = get_next_port(ports_yaml, category, service) - # Insert into the in-memory ports data under localhost - ports_yaml['ports']['localhost'].setdefault(category, {})[service] = port - assigned[entry] = port + for cat in args.ports: + loc = ports_yaml['ports']['localhost'].setdefault(cat, {}) + if app_id in loc: + print(f"→ Port for category '{cat}' and '{app_id}' already exists ({loc[app_id]}), skipping.") + continue + port = get_next_port(ports_yaml, cat) + loc[app_id] = port + assigned[cat] = port - # Backup and write updated all/09_ports.yml - backup_file = PORTS_FILE + '.bak' - shutil.copy(PORTS_FILE, backup_file) - dump_yaml(ports_yaml, PORTS_FILE) - print(f"→ Assigned ports: {assigned}. Updated {PORTS_FILE} (backup at {backup_file})") + if assigned: + shutil.copy(PORTS_FILE, PORTS_FILE + '.bak') + dump_yaml(ports_yaml, PORTS_FILE) + print(f"→ Assigned new ports: {assigned}") + else: + print("→ No new ports to assign, skipping update of 09_ports.yml.") - # Also write ports to the application’s own vars file - out_ports = {'ports': {'localhost': {}}} - for entry, port in assigned.items(): - category, service = entry.split('.', 1) - out_ports['ports']['localhost'].setdefault(category, {})[service] = port - ports_file = f'./group_vars/{app_id}_ports.yml' - dump_yaml(out_ports, ports_file) - print(f"→ Wrote assigned ports to {ports_file}") + # 4) Write or merge application-specific ports file + app_ports_file = f'./group_vars/{app_id}_ports.yml' + if os.path.exists(app_ports_file): + app_ports = load_yaml(app_ports_file) + dest = app_ports.setdefault('ports', {}).setdefault('localhost', {}) + for cat, port in assigned.items(): + dest[cat] = port + dump_yaml(app_ports, app_ports_file) + else: + dump_yaml({'ports': {'localhost': assigned}}, app_ports_file) + print(f"→ App-specific ports written to {app_ports_file}") if __name__ == '__main__': diff --git a/templates/docker_role/meta/main.yml.j2 b/templates/docker_role/meta/main.yml.j2 index 3a50b3fa..bcf77199 100644 --- a/templates/docker_role/meta/main.yml.j2 +++ b/templates/docker_role/meta/main.yml.j2 @@ -1,28 +1,20 @@ +{% raw %} --- galaxy_info: author: "Kevin Veen-Birkenbach" - description: "{{ description }}" + description: "" license: "CyMaIS NonCommercial License (CNCL)" license_url: "https://s.veen.world/cncl" company: | Kevin Veen-Birkenbach Consulting & Coaching Solutions https://www.veen.world - platforms: - - name: Docker - versions: - - latest - galaxy_tags: - {% for tag in tags %} - - {{ tag }} - {% endfor %} + galaxy_tags: [] repository: "https://github.com/kevinveenbirkenbach/cymais" issue_tracker_url: "https://github.com/kevinveenbirkenbach/cymais/issues" - documentation: "https://github.com/kevinveenbirkenbach/cymais/roles/{{application_id}}" + documentation: "https://github.com/kevinveenbirkenbach/cymais/" logo: - class: "{{ logo_classes }}" - run_after: - - docker-matomo - - docker-keycloak - - docker-mailu -dependencies: [] \ No newline at end of file + class: "" + run_after: [] +dependencies: [] +{% endraw %} \ No newline at end of file diff --git a/templates/docker_role/tasks/main.yml.j2 b/templates/docker_role/tasks/main.yml.j2 index 2079d6d1..723130e2 100644 --- a/templates/docker_role/tasks/main.yml.j2 +++ b/templates/docker_role/tasks/main.yml.j2 @@ -31,7 +31,7 @@ http_port: "{{ ports.localhost.http[application_id] }}" when: run_once_docker_{% endraw %}{{ application_id }}{% raw %} is not defined -- name: run the {% raw %}portfolio{% endraw %} tasks once +- name: run the {% endraw %}{{ application_id }}{% raw %} tasks once set_fact: run_once_docker_portfolio: true when: run_once_docker_{% endraw %}{{ application_id }}{% raw %} is not defined diff --git a/templates/docker_role/templates/docker-compose.yml.j2.j2 b/templates/docker_role/templates/docker-compose.yml.j2.j2 index 9ce42407..1afcb738 100644 --- a/templates/docker_role/templates/docker-compose.yml.j2.j2 +++ b/templates/docker_role/templates/docker-compose.yml.j2.j2 @@ -1,3 +1,4 @@ +{% raw %} services: {% include 'roles/docker-central-database/templates/services/main.yml.j2' %} @@ -13,7 +14,7 @@ services: {% include 'roles/docker-container/templates/networks.yml.j2' %} {% include 'roles/docker-compose/templates/volumes.yml.j2' %} - uploads: {% include 'roles/docker-compose/templates/networks.yml.j2' %} +{% endraw %} diff --git a/templates/docker_role/vars/configuration.yml.j2 b/templates/docker_role/vars/configuration.yml.j2 index 35d99617..d9700811 100644 --- a/templates/docker_role/vars/configuration.yml.j2 +++ b/templates/docker_role/vars/configuration.yml.j2 @@ -1,3 +1,4 @@ +{% raw %} credentials: docker: images: {} # @todo Move under services @@ -5,6 +6,8 @@ docker: services: redis: enabled: false # Enable Redis + database: + enabled: false # Enable the database features: matomo: true # Enable Matomo Tracking css: true # Enable Global CSS Styling @@ -24,3 +27,4 @@ rbac: mail-bot: description: "Has an token to send and recieve emails" +{% endraw %} \ No newline at end of file diff --git a/templates/docker_role/vars/main.yml.j2 b/templates/docker_role/vars/main.yml.j2 index 912a22c2..e93f0f8b 100644 --- a/templates/docker_role/vars/main.yml.j2 +++ b/templates/docker_role/vars/main.yml.j2 @@ -1,2 +1,2 @@ application_id: {{ application_id }} # ID of the application -database_type: {{ database }} # Database type [postgres, mariadb] \ No newline at end of file +database_type: 0 # Database type [postgres, mariadb] \ No newline at end of file