Optimized role creation script

This commit is contained in:
Kevin Veen-Birkenbach 2025-07-07 04:31:43 +02:00
parent 9762de2901
commit 634f1835fc
No known key found for this signature in database
GPG Key ID: 44D8F11FD62F878E
6 changed files with 131 additions and 79 deletions

View File

@ -2,14 +2,16 @@
import argparse import argparse
import os import os
import shutil import shutil
import sys
import yaml import yaml
import ipaddress import ipaddress
import difflib
from jinja2 import Environment, FileSystemLoader from jinja2 import Environment, FileSystemLoader
# Paths to the group-vars files # Paths to the group-vars files
PORTS_FILE = './group_vars/all/09_ports.yml' PORTS_FILE = './group_vars/all/09_ports.yml'
NETWORKS_FILE = './group_vars/all/10_networks.yml' NETWORKS_FILE = './group_vars/all/10_networks.yml'
ROLE_TEMPLATE_DIR = './docker-template' ROLE_TEMPLATE_DIR = './templates/docker_role'
ROLES_DIR = './roles' ROLES_DIR = './roles'
@ -24,35 +26,44 @@ def dump_yaml(data, path):
def get_next_network(networks_dict, prefixlen): def get_next_network(networks_dict, prefixlen):
# Collect all local subnets matching the given prefix length """Select the first available local subnet with the given prefix length."""
nets = [] nets = []
for name, info in networks_dict['defaults_networks']['local'].items(): for info in networks_dict['defaults_networks']['local'].values():
net = ipaddress.ip_network(info['subnet']) net = ipaddress.ip_network(info['subnet'])
if net.prefixlen == prefixlen: if net.prefixlen == prefixlen:
nets.append(net) nets.append(net)
# Sort by network address and return the first one
nets.sort(key=lambda n: int(n.network_address)) nets.sort(key=lambda n: int(n.network_address))
return nets[0] return nets[0]
def get_next_port(ports_dict, category, service): def get_next_port(ports_dict, category):
used = set() """Find the next unused port in the given localhost category."""
# Gather already taken ports under localhost.category used = {int(p) for p in ports_dict['ports']['localhost'].get(category, {}).values()}
for svc, port in ports_dict['ports']['localhost'].get(category, {}).items():
used.add(int(port))
# Start searching from port 1 upwards
candidate = 1 candidate = 1
while candidate in used: while candidate in used:
candidate += 1 candidate += 1
return candidate return candidate
def prompt_conflict(dst_file):
"""Prompt the user to resolve a file conflict."""
print(f"Conflict detected: {dst_file}")
print("Choose action: [1] overwrite, [2] skip, [3] merge")
choice = None
while choice not in ('1', '2', '3'):
choice = input("Enter 1, 2, or 3: ").strip()
return choice
def render_template(src_dir, dst_dir, context): def render_template(src_dir, dst_dir, context):
"""Recursively render all templates from src_dir into dst_dir, handling conflicts."""
env = Environment( env = Environment(
loader=FileSystemLoader(src_dir), loader=FileSystemLoader(src_dir),
keep_trailing_newline=True, keep_trailing_newline=True,
autoescape=False, autoescape=False,
) )
# Add a bool filter for Jinja evaluation
env.filters['bool'] = lambda x: bool(x)
for root, _, files in os.walk(src_dir): for root, _, files in os.walk(src_dir):
rel_path = os.path.relpath(root, src_dir) rel_path = os.path.relpath(root, src_dir)
target_path = os.path.join(dst_dir, rel_path) target_path = os.path.join(dst_dir, rel_path)
@ -61,83 +72,127 @@ def render_template(src_dir, dst_dir, context):
template = env.get_template(os.path.join(rel_path, filename)) template = env.get_template(os.path.join(rel_path, filename))
rendered = template.render(**context) rendered = template.render(**context)
out_name = filename[:-3] if filename.endswith('.j2') else filename out_name = filename[:-3] if filename.endswith('.j2') else filename
with open(os.path.join(target_path, out_name), 'w') as f: dst_file = os.path.join(target_path, out_name)
if os.path.exists(dst_file):
choice = prompt_conflict(dst_file)
if choice == '2':
print(f"Skipping {dst_file}")
continue
if choice == '3':
with open(dst_file) as f_old:
old_lines = f_old.readlines()
new_lines = rendered.splitlines(keepends=True)
diff = difflib.unified_diff(
old_lines, new_lines,
fromfile=f"a/{out_name}",
tofile=f"b/{out_name}",
lineterm=''
)
diff_path = dst_file + '.diff'
with open(diff_path, 'w') as fd:
fd.writelines(diff)
print(f"Diff written to {diff_path}; please merge manually.")
continue
# Overwrite
print(f"Overwriting {dst_file}")
with open(dst_file, 'w') as f:
f.write(rendered) f.write(rendered)
def main(): def main():
# Load current port categories dynamically
ports_yaml = load_yaml(PORTS_FILE)
categories = list(ports_yaml.get('ports', {}).get('localhost', {}).keys())
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Create a Docker Ansible role with Jinja2 templates, and assign network and ports" description="Create or update a Docker Ansible role, and assign network and ports"
) )
parser.add_argument( parser.add_argument(
'--application-id', '-a', required=True, '-a', '--application-id', required=True,
help="Unique ID of the application (used in the role name)" help="Unique application ID"
) )
parser.add_argument( parser.add_argument(
'--network', '-n', choices=['24', '28'], required=True, '-n', '--network', choices=['24', '28'], required=True,
help="Network prefix length to assign (/24 or /28)" help="Network prefix length (/24 or /28)"
) )
parser.add_argument( parser.add_argument(
'--ports', '-p', nargs='+', metavar="CATEGORY.SERVICE", required=True, '-p', '--ports', nargs='+', choices=categories, required=True,
help="List of ports in the format category.service (e.g. http.nextcloud)" help=f"Port categories to assign (allowed: {', '.join(categories)})"
) )
args = parser.parse_args() args = parser.parse_args()
app_id = args.application_id app_id = args.application_id
role_name = f"docker-{app_id}" role_name = f"docker-{app_id}"
# 1) Create the role from the template
role_dir = os.path.join(ROLES_DIR, role_name) role_dir = os.path.join(ROLES_DIR, role_name)
# If role directory exists, ask whether to continue
if os.path.exists(role_dir): if os.path.exists(role_dir):
parser.error(f"Role {role_name} already exists at {role_dir}") cont = input(f"Role {role_name} already exists. Continue updating? [y/N]: ").strip().lower()
if cont != 'y':
print("Aborting.")
sys.exit(1)
else:
os.makedirs(role_dir)
# 1) Render and copy templates, with conflict resolution
# Provide database_type=0 in context for task template logic
render_template(ROLE_TEMPLATE_DIR, role_dir, { render_template(ROLE_TEMPLATE_DIR, role_dir, {
'application_id': app_id, 'application_id': app_id,
'role_name': role_name, 'role_name': role_name,
'database_type': 0,
}) })
print(f"→ Role {role_name} created at {role_dir}") print(f"Templates rendered into {role_dir}")
# 2) Assign network # 2) Assign network if not already set
net_vars_file = f'./group_vars/{app_id}_network.yml'
if os.path.exists(net_vars_file):
existing_net = load_yaml(net_vars_file)
apps = existing_net.get('defaults_networks', {}).get('application', {})
if app_id in apps:
print(f"→ Network for {app_id} already configured ({apps[app_id]}), skipping.")
else:
networks = load_yaml(NETWORKS_FILE) networks = load_yaml(NETWORKS_FILE)
prefix = int(args.network) net = get_next_network(networks, int(args.network))
chosen_net = get_next_network(networks, prefix) apps[app_id] = str(net)
out_net = { dump_yaml(existing_net, net_vars_file)
'defaults_networks': { print(f"→ Appended network {net} for {app_id}")
'application': { else:
app_id: str(chosen_net) networks = load_yaml(NETWORKS_FILE)
} net = get_next_network(networks, int(args.network))
} dump_yaml({'defaults_networks': {'application': {app_id: str(net)}}}, net_vars_file)
} print(f"→ Created network vars file with {net} for {app_id}")
net_file = f'./group_vars/{app_id}_network.yml'
dump_yaml(out_net, net_file)
print(f"→ Assigned network {chosen_net} (/{prefix}) and wrote to {net_file}")
# 3) Assign ports # 3) Assign ports if not present
ports_yaml = load_yaml(PORTS_FILE)
assigned = {} assigned = {}
for entry in args.ports: for cat in args.ports:
try: loc = ports_yaml['ports']['localhost'].setdefault(cat, {})
category, service = entry.split('.', 1) if app_id in loc:
except ValueError: print(f"→ Port for category '{cat}' and '{app_id}' already exists ({loc[app_id]}), skipping.")
parser.error(f"Invalid port spec: {entry}. Must be CATEGORY.SERVICE") continue
port = get_next_port(ports_yaml, category, service) port = get_next_port(ports_yaml, cat)
# Insert into the in-memory ports data under localhost loc[app_id] = port
ports_yaml['ports']['localhost'].setdefault(category, {})[service] = port assigned[cat] = port
assigned[entry] = port
# Backup and write updated all/09_ports.yml if assigned:
backup_file = PORTS_FILE + '.bak' shutil.copy(PORTS_FILE, PORTS_FILE + '.bak')
shutil.copy(PORTS_FILE, backup_file)
dump_yaml(ports_yaml, PORTS_FILE) dump_yaml(ports_yaml, PORTS_FILE)
print(f"→ Assigned ports: {assigned}. Updated {PORTS_FILE} (backup at {backup_file})") print(f"→ Assigned new ports: {assigned}")
else:
print("→ No new ports to assign, skipping update of 09_ports.yml.")
# Also write ports to the applications own vars file # 4) Write or merge application-specific ports file
out_ports = {'ports': {'localhost': {}}} app_ports_file = f'./group_vars/{app_id}_ports.yml'
for entry, port in assigned.items(): if os.path.exists(app_ports_file):
category, service = entry.split('.', 1) app_ports = load_yaml(app_ports_file)
out_ports['ports']['localhost'].setdefault(category, {})[service] = port dest = app_ports.setdefault('ports', {}).setdefault('localhost', {})
ports_file = f'./group_vars/{app_id}_ports.yml' for cat, port in assigned.items():
dump_yaml(out_ports, ports_file) dest[cat] = port
print(f"→ Wrote assigned ports to {ports_file}") dump_yaml(app_ports, app_ports_file)
else:
dump_yaml({'ports': {'localhost': assigned}}, app_ports_file)
print(f"→ App-specific ports written to {app_ports_file}")
if __name__ == '__main__': if __name__ == '__main__':

View File

@ -1,28 +1,20 @@
{% raw %}
--- ---
galaxy_info: galaxy_info:
author: "Kevin Veen-Birkenbach" author: "Kevin Veen-Birkenbach"
description: "{{ description }}" description: ""
license: "CyMaIS NonCommercial License (CNCL)" license: "CyMaIS NonCommercial License (CNCL)"
license_url: "https://s.veen.world/cncl" license_url: "https://s.veen.world/cncl"
company: | company: |
Kevin Veen-Birkenbach Kevin Veen-Birkenbach
Consulting & Coaching Solutions Consulting & Coaching Solutions
https://www.veen.world https://www.veen.world
platforms: galaxy_tags: []
- name: Docker
versions:
- latest
galaxy_tags:
{% for tag in tags %}
- {{ tag }}
{% endfor %}
repository: "https://github.com/kevinveenbirkenbach/cymais" repository: "https://github.com/kevinveenbirkenbach/cymais"
issue_tracker_url: "https://github.com/kevinveenbirkenbach/cymais/issues" issue_tracker_url: "https://github.com/kevinveenbirkenbach/cymais/issues"
documentation: "https://github.com/kevinveenbirkenbach/cymais/roles/{{application_id}}" documentation: "https://github.com/kevinveenbirkenbach/cymais/"
logo: logo:
class: "{{ logo_classes }}" class: ""
run_after: run_after: []
- docker-matomo
- docker-keycloak
- docker-mailu
dependencies: [] dependencies: []
{% endraw %}

View File

@ -31,7 +31,7 @@
http_port: "{{ ports.localhost.http[application_id] }}" http_port: "{{ ports.localhost.http[application_id] }}"
when: run_once_docker_{% endraw %}{{ application_id }}{% raw %} is not defined when: run_once_docker_{% endraw %}{{ application_id }}{% raw %} is not defined
- name: run the {% raw %}portfolio{% endraw %} tasks once - name: run the {% endraw %}{{ application_id }}{% raw %} tasks once
set_fact: set_fact:
run_once_docker_portfolio: true run_once_docker_portfolio: true
when: run_once_docker_{% endraw %}{{ application_id }}{% raw %} is not defined when: run_once_docker_{% endraw %}{{ application_id }}{% raw %} is not defined

View File

@ -1,3 +1,4 @@
{% raw %}
services: services:
{% include 'roles/docker-central-database/templates/services/main.yml.j2' %} {% include 'roles/docker-central-database/templates/services/main.yml.j2' %}
@ -13,7 +14,7 @@ services:
{% include 'roles/docker-container/templates/networks.yml.j2' %} {% include 'roles/docker-container/templates/networks.yml.j2' %}
{% include 'roles/docker-compose/templates/volumes.yml.j2' %} {% include 'roles/docker-compose/templates/volumes.yml.j2' %}
uploads:
{% include 'roles/docker-compose/templates/networks.yml.j2' %} {% include 'roles/docker-compose/templates/networks.yml.j2' %}
{% endraw %}

View File

@ -1,3 +1,4 @@
{% raw %}
credentials: credentials:
docker: docker:
images: {} # @todo Move under services images: {} # @todo Move under services
@ -5,6 +6,8 @@ docker:
services: services:
redis: redis:
enabled: false # Enable Redis enabled: false # Enable Redis
database:
enabled: false # Enable the database
features: features:
matomo: true # Enable Matomo Tracking matomo: true # Enable Matomo Tracking
css: true # Enable Global CSS Styling css: true # Enable Global CSS Styling
@ -24,3 +27,4 @@ rbac:
mail-bot: mail-bot:
description: "Has an token to send and recieve emails" description: "Has an token to send and recieve emails"
{% endraw %}

View File

@ -1,2 +1,2 @@
application_id: {{ application_id }} # ID of the application application_id: {{ application_id }} # ID of the application
database_type: {{ database }} # Database type [postgres, mariadb] database_type: 0 # Database type [postgres, mariadb]