mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-10 20:56:39 +00:00
perf(sys-stk-front-proxy): write the body_filter Lua in the vhost's single contents call
Every front-proxy vhost wrote its body_filter Lua through a separate stack_host_template task in sys-front-inj-all: two remote round trips when unchanged (stat + file) and four to five when changed, 191 s over 54 vhosts per echoserver deploy and 179 s over 48 per galaxyserver deploy. The Lua now joins http.conf and the vhost in the Deploy task's stack_host_contents call, written first as before, rendered through template_shallow with application_id and domain pinned to the vhost. - sys-front-inj-all/tasks/main.yml no longer writes the Lua; the new standalone.yml includes it and keeps the separate write for matomo's re-inject path, which has no vhost Deploy task - stack_host_contents compares bytes instead of a universal-newline text read, so content with a CR stops reporting changed on every run and an existing non-UTF-8 file is replaced instead of crashing the module - a new required diff_paths argument keeps --diff for the Lua only; vhost and http.conf content stay out of the diff A Lua-only change still reports changed and fires reload-openresty once; the served probe and the fallback redeploy keep their old triggers. Check mode is unchanged. Tests: the real stack_host_template action against the fold expression across 99 roles and TLS/Tor/injection scenarios, byte-identical, with trim and newline negative controls; owner, group and mode parity with the copy module; byte compare with CR and non-UTF-8 files; the wiring of both callers. The saving (about 184 s and 174 s per deploy, net) is projected until the next deploys. 🎯 Fix: 92 % 💥 Break: 2 % 🌱 Efficiency: – ⚡ Speed: – 🛡️ Security: ~100 % 🏆 Score: 90 🚦 Gate: ✅ 90/5, efficiency and speed <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
kevinveenbirkenbachandClaude Opus 5.5 committed
1 parent
5556dabbb8
commit
4984f506c7
10 files changed
+1083
-26
No files matched your search
@@ -4,9 +4,13 @@ Args:
|
||||
files: list of ``{path, content}``; each path is written atomically when
|
||||
its content differs.
|
||||
mode: file mode applied to every path.
|
||||
diff_paths: the paths of ``files`` whose content change is reported in
|
||||
diff mode; the content of every other path stays out of the output.
|
||||
|
||||
Returns:
|
||||
changed_paths: the paths whose content or attributes changed.
|
||||
diff: in diff mode, one before/after entry per ``diff_paths`` path whose
|
||||
content differs.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
@@ -25,24 +29,38 @@ ARGUMENT_SPEC = {
|
||||
},
|
||||
},
|
||||
"mode": {"type": "raw", "required": True},
|
||||
"diff_paths": {"type": "list", "elements": "path", "required": True},
|
||||
}
|
||||
|
||||
|
||||
def read_current(target):
|
||||
if not target.exists():
|
||||
return None
|
||||
return target.read_bytes() # nocheck: cache-read managed host has no utils.cache
|
||||
|
||||
|
||||
def content_diff(path, content):
|
||||
current = read_current(Path(path))
|
||||
if current == content.encode("utf-8"):
|
||||
return None
|
||||
return {
|
||||
"before_header": path,
|
||||
"after_header": path,
|
||||
"before": (current or b"").decode("utf-8", errors="replace"),
|
||||
"after": content,
|
||||
}
|
||||
|
||||
|
||||
def write_file(module, path, content, mode):
|
||||
target = Path(path)
|
||||
current = None
|
||||
if target.exists():
|
||||
current = target.read_text(
|
||||
encoding="utf-8"
|
||||
) # nocheck: cache-read runs on the managed host, where utils.cache does not exist
|
||||
changed = current != content
|
||||
current = read_current(target)
|
||||
encoded = content.encode("utf-8")
|
||||
changed = current != encoded
|
||||
if module.check_mode and current is None:
|
||||
return changed
|
||||
if changed and not module.check_mode:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
"w", encoding="utf-8", dir=target.parent, delete=False
|
||||
) as handle:
|
||||
handle.write(content)
|
||||
with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as handle:
|
||||
handle.write(encoded)
|
||||
module.atomic_move(handle.name, str(target))
|
||||
file_args = module.load_file_common_arguments({"path": str(target), "mode": mode})
|
||||
return module.set_fs_attributes_if_different(file_args, changed)
|
||||
@@ -50,12 +68,22 @@ def write_file(module, path, content, mode):
|
||||
|
||||
def main():
|
||||
module = AnsibleModule(argument_spec=ARGUMENT_SPEC, supports_check_mode=True)
|
||||
changed_paths = [
|
||||
entry["path"]
|
||||
for entry in module.params["files"]
|
||||
if write_file(module, entry["path"], entry["content"], module.params["mode"])
|
||||
]
|
||||
module.exit_json(changed=bool(changed_paths), changed_paths=changed_paths)
|
||||
diff_paths = set(module.params["diff_paths"]) if module._diff else set()
|
||||
changed_paths = []
|
||||
diffs = []
|
||||
for entry in module.params["files"]:
|
||||
diff = (
|
||||
content_diff(entry["path"], entry["content"])
|
||||
if entry["path"] in diff_paths
|
||||
else None
|
||||
)
|
||||
if write_file(module, entry["path"], entry["content"], module.params["mode"]):
|
||||
changed_paths.append(entry["path"])
|
||||
if diff is not None:
|
||||
diffs.append(diff)
|
||||
module.exit_json(
|
||||
changed=bool(changed_paths), changed_paths=changed_paths, diff=diffs
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
---
|
||||
# nocheck: run-once runs once per vhost: every invocation re-applies the CDN dirs, inj-* and the body_filter Lua for the current `domain`.
|
||||
# nocheck: run-once runs once per vhost: every invocation re-applies the CDN dirs and the inj-* injection for the current `domain`.
|
||||
- name: 📂 Include dependency 'sys-svc-webserver-core'
|
||||
include_role:
|
||||
name: sys-svc-webserver-core
|
||||
@@ -43,10 +43,3 @@
|
||||
include_role:
|
||||
name: sys-front-inj-javascript
|
||||
when: inj_enabled.javascript
|
||||
|
||||
- name: "📄 Render body_filter Lua for '{{ domain }}'"
|
||||
stack_host_template:
|
||||
src: body_filter.lua.j2
|
||||
dest: "{{ lookup('nginx', 'files.lua', domain) }}"
|
||||
mode: "0644"
|
||||
notify: reload-openresty
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
# nocheck: run-once runs once per vhost of a caller that does not write the body_filter Lua itself.
|
||||
- name: "📂 Build the injection for '{{ domain }}'"
|
||||
ansible.builtin.include_tasks: main.yml
|
||||
|
||||
- name: "📄 Render body_filter Lua for '{{ domain }}'"
|
||||
stack_host_template:
|
||||
src: body_filter.lua.j2
|
||||
dest: "{{ lookup('nginx', 'files.lua', domain) }}"
|
||||
mode: "0644"
|
||||
notify: reload-openresty
|
||||
@@ -53,13 +53,19 @@
|
||||
- name: "🚀 Deploy NGINX config for '{{ front_proxy_domain }}'"
|
||||
stack_host_contents:
|
||||
files: >-
|
||||
{{ ([{'path': lookup('nginx', 'files.domain', front_proxy_domain, protocol='http'),
|
||||
{{ ([{'path': lookup('nginx', 'files.lua', front_proxy_domain),
|
||||
'content': lookup('template_shallow', lookup('path_absolute', 'roles/sys-front-inj-all/templates/body_filter.lua.j2'),
|
||||
template_vars={'application_id': front_proxy_application_id, 'domain': front_proxy_domain})}]
|
||||
if IS_STACK_HOST | bool else [])
|
||||
+ ([{'path': lookup('nginx', 'files.domain', front_proxy_domain, protocol='http'),
|
||||
'content': lookup('template', lookup('path_absolute', 'roles/sys-front-tls/templates/http.conf.j2'),
|
||||
template_vars={'domain': front_proxy_domain})}]
|
||||
if lookup('tls', front_proxy_domain, 'enabled') | bool else [])
|
||||
+ [{'path': front_proxy_domain_conf_dst_resolved,
|
||||
'content': lookup('template', front_proxy_template_src)}] }}
|
||||
mode: "0644"
|
||||
diff_paths:
|
||||
- "{{ lookup('nginx', 'files.lua', front_proxy_domain) }}"
|
||||
vars:
|
||||
application_id: "{{ front_proxy_application_id }}"
|
||||
domain: "{{ front_proxy_domain }}"
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
- name: "📄 Matomo | Re-render injection on every vhost of one web-svc consumer"
|
||||
ansible.builtin.include_role:
|
||||
name: sys-front-inj-all
|
||||
tasks_from: standalone.yml
|
||||
vars:
|
||||
application_id: "{{ matomo_inj_dir.matomo_inj_consumer.id }}"
|
||||
domain: "{{ matomo_inj_vhost }}"
|
||||
|
||||
@@ -84,4 +84,5 @@
|
||||
stack_host_contents:
|
||||
files: "{{ redirect_sources | map(attribute='files') | flatten }}"
|
||||
mode: "0644"
|
||||
diff_paths: []
|
||||
notify: reload-openresty
|
||||
Whitespace-only changes.
@@ -0,0 +1,564 @@
|
||||
"""The body_filter Lua the vhost write folds in is the file the template action writes.
|
||||
|
||||
``sys-front-inj-all/tasks/standalone.yml`` writes ``body_filter.lua`` through
|
||||
``stack_host_template``; ``sys-stk-front-proxy`` renders it through
|
||||
``lookup('template_shallow')`` and writes it in the same
|
||||
``stack_host_contents`` call as ``http.conf`` and the vhost. The renders run
|
||||
through Ansible's own template action and templar against the real lookups,
|
||||
filters and include tree, once in the variable scope each path really sees.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import re
|
||||
import tempfile
|
||||
import unittest
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
|
||||
from ansible.parsing.dataloader import DataLoader
|
||||
from ansible.plugins.loader import filter_loader
|
||||
from ansible.template import Templar, trust_as_template
|
||||
|
||||
from plugins.action.stack_host_template import ActionModule as StackHostTemplate
|
||||
from utils.cache import _reset_cache_for_tests
|
||||
from utils.cache.files import PROJECT_ROOT, iter_project_files, read_text
|
||||
from utils.cache.yaml import load_yaml_any
|
||||
from utils.domains.default_primary import default_domain_primary
|
||||
from utils.roles.mapping import (
|
||||
ROLE_FILE_DEFAULTS_MAIN,
|
||||
ROLE_FILE_TASKS_MAIN,
|
||||
ROLE_FILE_VARS_MAIN,
|
||||
)
|
||||
from utils.tls_common import iter_domains
|
||||
|
||||
ROLES = PROJECT_ROOT / "roles"
|
||||
FRONT = ROLES / "sys-stk-front-proxy"
|
||||
INJ = ROLES / "sys-front-inj-all"
|
||||
LUA_TEMPLATE = INJ / "templates" / "body_filter.lua.j2"
|
||||
DOMAIN = default_domain_primary()
|
||||
ONION = "b" * 56 + ".onion"
|
||||
STARTERS = re.compile(
|
||||
r"name:\s*['\"]?(sys-stk-full|sys-stk-front-proxy)['\"]?\s*$", re.MULTILINE
|
||||
)
|
||||
INJECTION_PROVIDERS = (
|
||||
"web-svc-design",
|
||||
"web-app-matomo",
|
||||
"web-svc-logout",
|
||||
"web-app-dashboard",
|
||||
)
|
||||
SCENARIOS = (
|
||||
(True, False, True),
|
||||
(False, False, False),
|
||||
(True, True, False),
|
||||
(False, True, True),
|
||||
)
|
||||
INCLUDE = re.compile(r"\{%-?\s*include\s+(?P<expr>[^%]+?)\s*-?%\}")
|
||||
LUA_ENTRY = (
|
||||
"([{'path': lookup('nginx', 'files.lua', front_proxy_domain), "
|
||||
"'content': lookup('template_shallow', lookup('path_absolute', "
|
||||
"'roles/sys-front-inj-all/templates/body_filter.lua.j2'), "
|
||||
"template_vars={'application_id': front_proxy_application_id, "
|
||||
"'domain': front_proxy_domain})}] "
|
||||
"if IS_STACK_HOST | bool else [])"
|
||||
)
|
||||
UNPINNED_ENTRY = LUA_ENTRY.replace(
|
||||
", template_vars={'application_id': front_proxy_application_id, "
|
||||
"'domain': front_proxy_domain}",
|
||||
"",
|
||||
)
|
||||
MIRROR_MARKER = b"do\n local pat = ([=[https://"
|
||||
RICH = {
|
||||
"app_style_present": True,
|
||||
"app_style_file": "style.css",
|
||||
"app_style_css": "body { color: #123; }\n",
|
||||
"design_js_one_liner": "(function(){var s='\\\\n';window.__x=\"]=] ä ✓\";})();",
|
||||
"design_config_json": '{"title":"<A & B>","q":"\\"x\\""}',
|
||||
"logout_code_one_liner": "document.addEventListener('click',function(e){}); ",
|
||||
"iframe_init_code_one_liner": "window.parent.postMessage({h:1},'*');",
|
||||
"javascript_code_one_liner": "console.log('\\t{{ not a template }}');",
|
||||
"matomo_site_id": "7",
|
||||
"matomo_tracking_code_one_liner": "var _paq=window._paq=window._paq||[];",
|
||||
}
|
||||
SPARSE = {
|
||||
**dict.fromkeys(RICH, ""),
|
||||
"app_style_present": False,
|
||||
}
|
||||
EDGE_TEMPLATES = {
|
||||
"one_newline.j2": "a\n",
|
||||
"two_newlines.j2": "a\n\n",
|
||||
"no_newline.j2": "a",
|
||||
"only_newlines.j2": "\n\n\n",
|
||||
"blocks.j2": (
|
||||
"{% if flag %}\n yes\n{% endif %}\n"
|
||||
" {% for i in [1, 2] %}\n{{ i }}\n {% endfor %}\n"
|
||||
"{%- if flag %} tight {% endif -%}\n"
|
||||
"{# comment #}\nend\n"
|
||||
),
|
||||
"macro.j2": (
|
||||
"{% macro push(name) -%}\n{{ name }}[1] = [=[\n"
|
||||
" {%- include 'one_newline.j2' -%}\n]=]\n{% endmacro %}\n"
|
||||
"{{ push('top') }}\n{{ push('head') }}"
|
||||
),
|
||||
"carriage.j2": "a\r\nb\r\n{{ value }}\r\n",
|
||||
"unicode.j2": "ä ✓ {{ value }}\t\n",
|
||||
"backslash.j2": "x\\ny {{ '\\\\' }} \\\\ end\n",
|
||||
}
|
||||
|
||||
|
||||
def _trusted(value):
|
||||
if isinstance(value, str):
|
||||
return trust_as_template(value)
|
||||
if isinstance(value, Mapping):
|
||||
return {key: _trusted(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_trusted(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _yaml(path: Path) -> dict:
|
||||
data = load_yaml_any(str(path), default_if_missing={})
|
||||
if not isinstance(data, Mapping):
|
||||
return {}
|
||||
return {key: _trusted(value) for key, value in data.items()}
|
||||
|
||||
|
||||
def _task_with(path: Path, module: str) -> dict:
|
||||
return next(t for t in load_yaml_any(str(path)) if module in t)
|
||||
|
||||
|
||||
def _normalised(text: str) -> str:
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
class _Shell:
|
||||
tmpdir = "/nonexistent/remote/tmp"
|
||||
|
||||
|
||||
class _Connection:
|
||||
_shell = _Shell()
|
||||
|
||||
|
||||
class _Task:
|
||||
async_val = 0
|
||||
check_mode = False
|
||||
|
||||
def __init__(self, args: dict, search_path: list[str]) -> None:
|
||||
self.args = dict(args)
|
||||
self._search_path = list(search_path)
|
||||
|
||||
def get_search_path(self) -> list[str]:
|
||||
return list(self._search_path)
|
||||
|
||||
def copy(self) -> _Task:
|
||||
return _Task(self.args, self._search_path)
|
||||
|
||||
|
||||
class _CapturingCopy:
|
||||
def __init__(self, task: _Task) -> None:
|
||||
self._task = task
|
||||
|
||||
def run(self, task_vars=None) -> dict:
|
||||
source = Path(self._task.args["src"])
|
||||
return {
|
||||
"bytes": source.read_bytes(), # nocheck: cache-read the action's own temp render, deleted right after
|
||||
"dest": self._task.args["dest"],
|
||||
"mode": self._task.args["mode"],
|
||||
}
|
||||
|
||||
|
||||
class _ActionLoader:
|
||||
def get(self, name: str, task: _Task, **kwargs) -> _CapturingCopy:
|
||||
assert name == "ansible.legacy.copy", name
|
||||
return _CapturingCopy(task)
|
||||
|
||||
|
||||
class _SharedLoader:
|
||||
action_loader = _ActionLoader()
|
||||
|
||||
|
||||
def render_action(task_vars: dict, src: str, dest: str, search_path, **args) -> dict:
|
||||
"""Run the real ``stack_host_template`` action up to the copy it hands off to.
|
||||
|
||||
Args:
|
||||
task_vars: the variables the task sees.
|
||||
src: the ``src`` argument, resolved like the task resolves it.
|
||||
dest: the ``dest`` argument.
|
||||
search_path: the task's role search path.
|
||||
args: further template action arguments, e.g. ``lstrip_blocks``.
|
||||
|
||||
Returns:
|
||||
``{"bytes", "dest", "mode"}`` the copy module would receive.
|
||||
"""
|
||||
loader = DataLoader()
|
||||
loader.set_basedir(str(PROJECT_ROOT))
|
||||
action = StackHostTemplate(
|
||||
task=_Task({"src": src, "dest": dest, **args}, search_path),
|
||||
connection=_Connection(),
|
||||
play_context=None,
|
||||
loader=loader,
|
||||
templar=Templar(loader=loader),
|
||||
shared_loader_obj=None,
|
||||
)
|
||||
action._shared_loader_obj = _SharedLoader()
|
||||
return action.run(task_vars=task_vars)
|
||||
|
||||
|
||||
def render_expression(variables: dict, expression: str):
|
||||
loader = DataLoader()
|
||||
loader.set_basedir(str(PROJECT_ROOT))
|
||||
return Templar(loader=loader, variables=variables).template(
|
||||
trust_as_template(expression)
|
||||
)
|
||||
|
||||
|
||||
def _outcome(render):
|
||||
try:
|
||||
return ("ok", render())
|
||||
except Exception as error:
|
||||
causes = []
|
||||
while error is not None:
|
||||
causes.append(str(error).replace("template_shallow", "template"))
|
||||
error = error.__cause__
|
||||
return ("error", causes[-1])
|
||||
|
||||
|
||||
def front_proxy_roles() -> list[str]:
|
||||
"""Return every role whose vhost ``sys-stk-front-proxy`` writes."""
|
||||
found = set()
|
||||
for name in iter_project_files(extensions=(".yml", ".j2"), exclude_tests=True):
|
||||
path = Path(name)
|
||||
if not path.is_relative_to(ROLES):
|
||||
continue
|
||||
parts = path.relative_to(ROLES).parts
|
||||
if len(parts) < 3 or parts[0] in ("sys-stk-full", FRONT.name):
|
||||
continue
|
||||
if parts[1:] == ("templates", "proxy.conf.j2") or (
|
||||
parts[1] == "tasks"
|
||||
and path.suffix == ".yml"
|
||||
and STARTERS.search(read_text(name))
|
||||
):
|
||||
found.add(parts[0])
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def lua_include_tree() -> list[Path]:
|
||||
"""Return the Lua template and every template its includes can reach."""
|
||||
seen: list[Path] = []
|
||||
pending = [LUA_TEMPLATE]
|
||||
while pending:
|
||||
path = pending.pop()
|
||||
if path in seen:
|
||||
continue
|
||||
seen.append(path)
|
||||
for match in INCLUDE.finditer(read_text(str(path))):
|
||||
expr = match["expr"]
|
||||
if "~ f ~" in expr:
|
||||
pending.extend(sorted(ROLES.glob("sys-front-inj-*/templates/*_sub.j2")))
|
||||
else:
|
||||
pending.extend(
|
||||
PROJECT_ROOT / name for name in re.findall(r"'([^']+\.j2)'", expr)
|
||||
)
|
||||
return seen
|
||||
|
||||
|
||||
class _Harness:
|
||||
def __init__(self) -> None:
|
||||
self.group_vars: dict = {}
|
||||
for path in sorted((PROJECT_ROOT / "group_vars" / "all").glob("*.yml")):
|
||||
self.group_vars.update(_yaml(path))
|
||||
vhost_tasks = FRONT / "tasks" / "vhost.yml"
|
||||
self.deploy = _task_with(vhost_tasks, "stack_host_contents")
|
||||
self.main = _task_with(FRONT / ROLE_FILE_TASKS_MAIN, "ansible.builtin.set_fact")
|
||||
self.inj = _task_with(INJ / ROLE_FILE_TASKS_MAIN, "set_fact")
|
||||
self.lua_task = _task_with(
|
||||
INJ / "tasks" / "standalone.yml", "stack_host_template"
|
||||
)
|
||||
filter_loader.add_directory(str(INJ / "filter_plugins"))
|
||||
|
||||
def base(self, app: str, tls: bool, tor: bool, providers: bool) -> dict:
|
||||
groups = (
|
||||
["svc-prx-openresty"]
|
||||
+ (["svc-net-tor"] if tor else [])
|
||||
+ (list(INJECTION_PROVIDERS) if providers else [])
|
||||
)
|
||||
return {
|
||||
**self.group_vars,
|
||||
**_yaml(FRONT / ROLE_FILE_DEFAULTS_MAIN),
|
||||
**_yaml(ROLES / app / ROLE_FILE_DEFAULTS_MAIN),
|
||||
**_yaml(ROLES / app / ROLE_FILE_VARS_MAIN),
|
||||
"playbook_dir": str(PROJECT_ROOT),
|
||||
"inventory_hostname": "localhost",
|
||||
"group_names": groups,
|
||||
"groups": {group: ["localhost"] for group in groups},
|
||||
"DOMAIN_PRIMARY": DOMAIN,
|
||||
"SYSTEM_EMAIL_DOMAIN": DOMAIN,
|
||||
"TLS_ENABLED": tls,
|
||||
"applications": (
|
||||
{"svc-net-tor": {"services": {"tor": {"node": ONION}}}} if tor else {}
|
||||
),
|
||||
"users": {},
|
||||
"IS_STACK_HOST": True,
|
||||
"RESOURCE_HOST_CPUS_OVERRIDE": 8,
|
||||
"RESOURCE_HOST_MEM_MB_OVERRIDE": 32768,
|
||||
"application_id": app,
|
||||
}
|
||||
|
||||
def vhosts(self, variables: dict, app: str) -> list[str]:
|
||||
domains = render_expression(variables, "{{ lookup('domains') }}").get(app)
|
||||
found: list[str] = []
|
||||
for domain in iter_domains(domains) if domains else []:
|
||||
for vhost in render_expression(
|
||||
variables, f"{{{{ lookup('network_siblings', '{domain}') }}}}"
|
||||
):
|
||||
if vhost not in found:
|
||||
found.append(vhost)
|
||||
return found
|
||||
|
||||
def scopes(self, app: str, tls: bool, tor: bool, providers: bool, facts: dict):
|
||||
"""Yield ``(vhost, action_task_vars, fold_task_vars)`` per vhost of ``app``.
|
||||
|
||||
Args:
|
||||
app: the application whose vhosts are rendered.
|
||||
tls: the inventory ``TLS_ENABLED``.
|
||||
tor: whether ``svc-net-tor`` serves the host with a node onion.
|
||||
providers: whether the injection providers run on the host.
|
||||
facts: the injection facts the ``sys-front-inj-*`` roles set.
|
||||
"""
|
||||
variables = self.base(app, tls, tor, providers)
|
||||
variables.update(_yaml(FRONT / ROLE_FILE_VARS_MAIN))
|
||||
main_facts = Templar(loader=DataLoader(), variables=variables).template(
|
||||
_trusted(self.main["ansible.builtin.set_fact"])
|
||||
)
|
||||
for vhost in self.vhosts(variables, app):
|
||||
shared = {
|
||||
**variables,
|
||||
**main_facts,
|
||||
"front_proxy_domain": vhost,
|
||||
"domain": vhost,
|
||||
}
|
||||
inj_scope = {
|
||||
**shared,
|
||||
**_yaml(INJ / ROLE_FILE_DEFAULTS_MAIN),
|
||||
**_yaml(INJ / ROLE_FILE_VARS_MAIN),
|
||||
}
|
||||
inj_facts = Templar(loader=DataLoader(), variables=inj_scope).template(
|
||||
_trusted(self.inj["set_fact"])
|
||||
)
|
||||
shared.update(inj_facts)
|
||||
shared.update(facts)
|
||||
for feature in ("logout", "dashboard"):
|
||||
if inj_facts["inj_enabled"].get(feature):
|
||||
shared.update(
|
||||
_yaml(
|
||||
ROLES / f"sys-front-inj-{feature}" / ROLE_FILE_DEFAULTS_MAIN
|
||||
)
|
||||
)
|
||||
shared.update(
|
||||
_yaml(ROLES / f"sys-front-inj-{feature}" / ROLE_FILE_VARS_MAIN)
|
||||
)
|
||||
action_vars = {
|
||||
**shared,
|
||||
**_yaml(INJ / ROLE_FILE_DEFAULTS_MAIN),
|
||||
**_yaml(INJ / ROLE_FILE_VARS_MAIN),
|
||||
"ansible_search_path": [str(INJ), str(PROJECT_ROOT)],
|
||||
}
|
||||
fold_vars = {
|
||||
**shared,
|
||||
**_trusted(self.deploy["vars"]),
|
||||
"ansible_search_path": [str(FRONT), str(PROJECT_ROOT)],
|
||||
}
|
||||
yield vhost, action_vars, fold_vars
|
||||
|
||||
def render_pair(self, action_vars: dict, fold_vars: dict):
|
||||
def action():
|
||||
dest = render_expression(
|
||||
action_vars, self.lua_task["stack_host_template"]["dest"]
|
||||
)
|
||||
result = render_action(
|
||||
action_vars,
|
||||
self.lua_task["stack_host_template"]["src"],
|
||||
dest,
|
||||
[str(INJ), str(INJ / "tasks")],
|
||||
mode=self.lua_task["stack_host_template"]["mode"],
|
||||
)
|
||||
return (result["dest"], result["bytes"], result["mode"])
|
||||
|
||||
return _outcome(action), self.fold(fold_vars, LUA_ENTRY)
|
||||
|
||||
def fold(self, fold_vars: dict, entry: str):
|
||||
def render():
|
||||
[rendered] = render_expression(fold_vars, "{{ " + entry + " }}")
|
||||
return (
|
||||
rendered["path"],
|
||||
rendered["content"].encode("utf-8"),
|
||||
self.deploy["stack_host_contents"]["mode"],
|
||||
)
|
||||
|
||||
return _outcome(render)
|
||||
|
||||
|
||||
class TestTheFoldRendersTheExactCallSite(unittest.TestCase):
|
||||
def test_the_deploy_task_carries_the_rendered_lua_entry(self) -> None:
|
||||
harness = _Harness()
|
||||
files = harness.deploy["stack_host_contents"]["files"]
|
||||
self.assertIn(_normalised(LUA_ENTRY), _normalised(files))
|
||||
|
||||
def test_no_template_of_the_lua_tree_reads_template_action_only_vars(self) -> None:
|
||||
tree = lua_include_tree()
|
||||
self.assertGreater(len(tree), 9)
|
||||
for path in tree:
|
||||
with self.subTest(template=str(path.relative_to(PROJECT_ROOT))):
|
||||
self.assertNotRegex(
|
||||
read_text(str(path)),
|
||||
r"\btemplate_(destpath|host|uid|path|fullpath|mtime|run_date)\b|\bansible_managed\b",
|
||||
)
|
||||
|
||||
|
||||
class TestEdgeCaseTemplatesRenderAlike(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls._tmp = tempfile.TemporaryDirectory()
|
||||
cls.root = Path(cls._tmp.name)
|
||||
(cls.root / "templates").mkdir()
|
||||
for name, body in EDGE_TEMPLATES.items():
|
||||
(cls.root / "templates" / name).write_bytes(body.encode("utf-8"))
|
||||
cls.variables = {
|
||||
"flag": True,
|
||||
"value": "v\\1 ]=] ä",
|
||||
"IS_STACK_HOST": True,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cls._tmp.cleanup()
|
||||
|
||||
def _action(self, name: str, **args) -> bytes:
|
||||
task_vars = {**self.variables, "ansible_search_path": [str(self.root)]}
|
||||
return render_action(task_vars, name, "/x/" + name, [str(self.root)], **args)[
|
||||
"bytes"
|
||||
]
|
||||
|
||||
def _lookup(self, name: str, plugin: str = "template_shallow") -> bytes:
|
||||
variables = {**self.variables, "ansible_search_path": [str(self.root)]}
|
||||
path = self.root / "templates" / name
|
||||
return render_expression(
|
||||
variables, f"{{{{ lookup('{plugin}', '{path}') }}}}"
|
||||
).encode("utf-8")
|
||||
|
||||
def test_every_edge_case_renders_byte_identical(self) -> None:
|
||||
for name in EDGE_TEMPLATES:
|
||||
with self.subTest(template=name):
|
||||
action = self._action(name)
|
||||
self.assertEqual(self._lookup(name), action)
|
||||
self.assertEqual(self._lookup(name, "template"), action)
|
||||
|
||||
def test_the_trailing_newlines_survive_both_paths(self) -> None:
|
||||
self.assertEqual(self._lookup("one_newline.j2"), b"a\n")
|
||||
self.assertEqual(self._lookup("two_newlines.j2"), b"a\n\n")
|
||||
self.assertEqual(self._lookup("no_newline.j2"), b"a")
|
||||
|
||||
def test_a_whitespace_semantics_difference_is_detected(self) -> None:
|
||||
self.assertNotEqual(
|
||||
self._action("blocks.j2", lstrip_blocks=True), self._lookup("blocks.j2")
|
||||
)
|
||||
self.assertNotEqual(
|
||||
self._action("blocks.j2", trim_blocks=False), self._lookup("blocks.j2")
|
||||
)
|
||||
|
||||
|
||||
class TestEveryRoleRendersTheSameLua(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.roles = front_proxy_roles()
|
||||
cls.harness = harness = _Harness()
|
||||
cls.cases = []
|
||||
cls.mutated = []
|
||||
cls.mirrored = []
|
||||
for tls, tor, providers in SCENARIOS:
|
||||
_reset_cache_for_tests()
|
||||
index = 0
|
||||
for app in cls.roles:
|
||||
for vhost, action_vars, fold_vars in harness.scopes(
|
||||
app, tls, tor, providers, RICH if index % 2 == 0 else SPARSE
|
||||
):
|
||||
snapshot = copy.deepcopy(fold_vars)
|
||||
action, fold = harness.render_pair(action_vars, fold_vars)
|
||||
if fold_vars != snapshot:
|
||||
cls.mutated.append((app, vhost, tls, tor, providers))
|
||||
case = (app, vhost, tls, tor, providers, index % 2)
|
||||
cls.cases.append((case, action, fold))
|
||||
if (
|
||||
action[0] == "ok"
|
||||
and MIRROR_MARKER in action[1][1]
|
||||
and len(cls.mirrored) < 3
|
||||
):
|
||||
cls.mirrored.append((case, action, fold_vars))
|
||||
index += 1
|
||||
_reset_cache_for_tests()
|
||||
|
||||
def test_the_scan_finds_every_front_proxy_role(self) -> None:
|
||||
self.assertGreater(len(self.roles), 90)
|
||||
self.assertIn("web-app-wordpress", self.roles)
|
||||
self.assertIn("web-svc-html", self.roles)
|
||||
|
||||
def test_the_action_and_the_fold_agree_byte_for_byte(self) -> None:
|
||||
for case, action, fold in self.cases:
|
||||
with self.subTest(case=case):
|
||||
self.assertEqual(fold, action)
|
||||
|
||||
def test_almost_every_case_renders(self) -> None:
|
||||
rendered = sum(action[0] == "ok" for _, action, _ in self.cases)
|
||||
self.assertGreater(rendered, 0.95 * len(self.cases))
|
||||
self.assertGreater(rendered, 4 * len(self.roles))
|
||||
|
||||
def test_the_cases_reach_every_branch_of_the_template(self) -> None:
|
||||
bodies = [
|
||||
action[1][1].decode("utf-8")
|
||||
for _, action, _ in self.cases
|
||||
if action[0] == "ok"
|
||||
]
|
||||
onion = [
|
||||
c for c, a, _ in self.cases if c[1].endswith(".onion") and a[0] == "ok"
|
||||
]
|
||||
self.assertTrue(onion)
|
||||
for marker in (
|
||||
"top_snippets[#top_snippets + 1]",
|
||||
"head_snippets[#head_snippets + 1]",
|
||||
"body_snippets[#body_snippets + 1]",
|
||||
'host:sub(-6) == ".onion"',
|
||||
MIRROR_MARKER.decode("utf-8"),
|
||||
"data-design=",
|
||||
"?version=",
|
||||
):
|
||||
with self.subTest(marker=marker):
|
||||
self.assertTrue(any(marker in body for body in bodies))
|
||||
self.assertTrue(any("data-design=" not in body for body in bodies))
|
||||
|
||||
def test_the_shallow_render_leaves_every_variable_untouched(self) -> None:
|
||||
self.assertEqual(self.mutated, [])
|
||||
|
||||
def test_every_render_compares_alike_as_text_and_as_bytes(self) -> None:
|
||||
for case, action, _ in self.cases:
|
||||
if action[0] == "ok":
|
||||
with self.subTest(case=case):
|
||||
self.assertNotIn(b"\r", action[1][1])
|
||||
action[1][1].decode("utf-8")
|
||||
|
||||
def test_an_outranking_include_param_cannot_reach_the_lua(self) -> None:
|
||||
self.assertEqual(len(self.mirrored), 3)
|
||||
for case, action, fold_vars in self.mirrored:
|
||||
_reset_cache_for_tests()
|
||||
for drift in ({"domain": DOMAIN}, {"application_id": "web-svc-html"}):
|
||||
drifted = {**fold_vars, **drift}
|
||||
with self.subTest(case=case, drift=drift):
|
||||
self.assertEqual(self.harness.fold(drifted, LUA_ENTRY), action)
|
||||
self.assertNotEqual(
|
||||
self.harness.fold(drifted, UNPINNED_ENTRY), action
|
||||
)
|
||||
_reset_cache_for_tests()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,6 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -10,6 +14,9 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
from ansible.module_utils.testing import patch_module_args
|
||||
from ansible.modules import copy as copy_module
|
||||
|
||||
from utils.cache.files import read_text
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
@@ -19,6 +26,13 @@ if TYPE_CHECKING:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
MODULE_PATH = PROJECT_ROOT / "library" / "stack_host_contents.py"
|
||||
ROLES = PROJECT_ROOT / "roles"
|
||||
TODAY_TEMPLATES = (
|
||||
ROLES / "sys-front-tls" / "templates" / "http.conf.j2",
|
||||
ROLES / "sys-front-inj-all" / "templates" / "body_filter.lua.j2",
|
||||
*sorted((ROLES / "sys-svc-proxy" / "templates" / "vhost").glob("*.conf.j2")),
|
||||
*sorted(ROLES.glob("*/templates/proxy.conf.j2")),
|
||||
)
|
||||
|
||||
|
||||
def _load_stack_host_contents() -> ModuleType:
|
||||
@@ -44,7 +58,12 @@ class WriteFileTests(unittest.TestCase):
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _module(self, *, check_mode=False):
|
||||
args = {"files": [], "mode": "0644", "_ansible_check_mode": check_mode}
|
||||
args = {
|
||||
"files": [],
|
||||
"mode": "0644",
|
||||
"diff_paths": [],
|
||||
"_ansible_check_mode": check_mode,
|
||||
}
|
||||
with patch_module_args(args):
|
||||
return AnsibleModule(
|
||||
argument_spec=_MOD.ARGUMENT_SPEC, supports_check_mode=True
|
||||
@@ -95,6 +114,251 @@ class WriteFileTests(unittest.TestCase):
|
||||
self._write("other\n")
|
||||
self.assertEqual([p.name for p in self.path.parent.iterdir()], [self.path.name])
|
||||
|
||||
def _bytes(self):
|
||||
return self.path.read_bytes() # nocheck: cache-read rewritten between reads
|
||||
|
||||
def test_carriage_returns_are_stable_on_the_second_run(self):
|
||||
content = "a\r\nb\rc\n"
|
||||
self.assertTrue(self._write(content))
|
||||
self.assertEqual(self._bytes(), content.encode("utf-8"))
|
||||
self.assertFalse(self._write(content))
|
||||
|
||||
def test_a_text_read_turns_carriage_returns_into_a_change(self):
|
||||
self._write("a\r\nb\n")
|
||||
self.assertNotEqual(self._content(), "a\r\nb\n")
|
||||
|
||||
def test_a_non_utf8_file_is_replaced_without_a_crash(self):
|
||||
self.path.write_bytes(b"\xff\xfe\xe4 latin-1\n")
|
||||
with self.assertRaises(UnicodeDecodeError):
|
||||
self._content()
|
||||
self.assertTrue(self._write("server {}\n"))
|
||||
self.assertEqual(self._bytes(), b"server {}\n")
|
||||
self.assertFalse(self._write("server {}\n"))
|
||||
|
||||
def test_todays_templates_written_as_text_stay_unchanged(self):
|
||||
self.assertGreater(len(TODAY_TEMPLATES), 20)
|
||||
for source in TODAY_TEMPLATES:
|
||||
content = read_text(str(source))
|
||||
with self.subTest(template=str(source.relative_to(PROJECT_ROOT))):
|
||||
self.path.write_text(content, encoding="utf-8")
|
||||
self.assertEqual(self._content(), content)
|
||||
self.assertFalse(self._write(content))
|
||||
|
||||
|
||||
def _run_module(main, args: dict) -> dict:
|
||||
output = io.StringIO()
|
||||
with (
|
||||
patch_module_args(args),
|
||||
contextlib.redirect_stdout(output),
|
||||
contextlib.suppress(SystemExit),
|
||||
):
|
||||
main()
|
||||
return json.loads(output.getvalue())
|
||||
|
||||
|
||||
def _run_main(args: dict) -> dict:
|
||||
return _run_module(_MOD.main, args)
|
||||
|
||||
|
||||
class MainTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
self.lua = self.root / "body.lua"
|
||||
self.vhost = self.root / "vhost.conf"
|
||||
|
||||
def tearDown(self):
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _run(self, lua: str, vhost: str, *, diff=True, check=False) -> dict:
|
||||
return _run_main(
|
||||
{
|
||||
"files": [
|
||||
{"path": str(self.lua), "content": lua},
|
||||
{"path": str(self.vhost), "content": vhost},
|
||||
],
|
||||
"mode": "0644",
|
||||
"diff_paths": [str(self.lua)],
|
||||
"_ansible_diff": diff,
|
||||
"_ansible_check_mode": check,
|
||||
}
|
||||
)
|
||||
|
||||
def test_a_lua_only_change_is_changed_and_names_only_the_lua(self):
|
||||
self._run("a\n", "v\n")
|
||||
result = self._run("b\n", "v\n")
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertEqual(result["changed_paths"], [str(self.lua)])
|
||||
|
||||
def test_an_unchanged_run_reports_ok(self):
|
||||
self._run("a\n", "v\n")
|
||||
result = self._run("a\n", "v\n")
|
||||
self.assertFalse(result["changed"])
|
||||
self.assertEqual(result["changed_paths"], [])
|
||||
self.assertEqual(result["diff"], [])
|
||||
|
||||
def test_diff_mode_reports_only_the_listed_path(self):
|
||||
self._run("a\n", "v\n")
|
||||
result = self._run("b\n", "secret\n")
|
||||
self.assertEqual(
|
||||
result["diff"],
|
||||
[
|
||||
{
|
||||
"before_header": str(self.lua),
|
||||
"after_header": str(self.lua),
|
||||
"before": "a\n",
|
||||
"after": "b\n",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
def test_a_new_file_diffs_from_empty(self):
|
||||
result = self._run("a\n", "v\n")
|
||||
self.assertEqual(result["diff"][0]["before"], "")
|
||||
self.assertEqual(result["diff"][0]["after"], "a\n")
|
||||
|
||||
def test_without_diff_mode_no_content_leaves_the_module(self):
|
||||
self._run("a\n", "v\n", diff=False)
|
||||
result = self._run("b\n", "w\n", diff=False)
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertEqual(result["diff"], [])
|
||||
|
||||
def test_check_mode_diffs_without_writing(self):
|
||||
self._run("a\n", "v\n")
|
||||
result = self._run("b\n", "v\n", check=True)
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertEqual(result["diff"][0]["after"], "b\n")
|
||||
self.assertEqual(
|
||||
self.lua.read_text(
|
||||
encoding="utf-8"
|
||||
), # nocheck: cache-read the file under test changes between reads
|
||||
"a\n",
|
||||
)
|
||||
|
||||
def test_carriage_returns_neither_change_nor_diff_on_the_second_run(self):
|
||||
self._run("a\r\n", "v\r\n")
|
||||
result = self._run("a\r\n", "v\r\n")
|
||||
self.assertFalse(result["changed"])
|
||||
self.assertEqual(result["diff"], [])
|
||||
|
||||
def test_a_non_utf8_file_diffs_with_replacement_characters(self):
|
||||
self.lua.write_bytes(b"old \xff\n")
|
||||
self.vhost.write_bytes(b"\xe4\n")
|
||||
result = self._run("new\n", "v\n")
|
||||
self.assertEqual(result["changed_paths"], [str(self.lua), str(self.vhost)])
|
||||
self.assertEqual(result["diff"][0]["before"], "old �\n")
|
||||
|
||||
def test_the_diff_paths_argument_is_required(self):
|
||||
result = _run_main(
|
||||
{"files": [{"path": str(self.lua), "content": "a\n"}], "mode": "0644"}
|
||||
)
|
||||
self.assertTrue(result["failed"])
|
||||
self.assertIn("diff_paths", result["msg"])
|
||||
|
||||
|
||||
class AttributeParityWithTheCopyModuleTests(unittest.TestCase):
|
||||
"""The template action ends in the copy module; the fold ends in write_file."""
|
||||
|
||||
CONTENT = 'ngx.arg[1] = whole or ""\n'
|
||||
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
self.group = self._assignable_secondary_group()
|
||||
|
||||
def _assignable_secondary_group(self) -> int:
|
||||
probe = self.root / "probe"
|
||||
probe.touch()
|
||||
for gid in os.getgroups():
|
||||
if gid == os.getegid():
|
||||
continue
|
||||
try:
|
||||
os.chown(probe, -1, gid)
|
||||
except OSError:
|
||||
continue
|
||||
return gid
|
||||
return os.getegid()
|
||||
|
||||
def tearDown(self):
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _twin_dirs(self, name: str, *, setgid: bool) -> tuple[Path, Path]:
|
||||
dirs = (self.root / f"{name}-copy", self.root / f"{name}-fold")
|
||||
for directory in dirs:
|
||||
directory.mkdir()
|
||||
if setgid:
|
||||
os.chown(directory, -1, self.group)
|
||||
directory.chmod(0o2775)
|
||||
return dirs
|
||||
|
||||
def _copy(self, dest: Path) -> None:
|
||||
staged = self.root / f"staged-{dest.parent.name}"
|
||||
staged.write_text(self.CONTENT, encoding="utf-8")
|
||||
result = _run_module(
|
||||
copy_module.main,
|
||||
{
|
||||
"src": str(staged),
|
||||
"dest": str(dest),
|
||||
"mode": "0644",
|
||||
"_original_basename": "body_filter.lua.j2",
|
||||
},
|
||||
)
|
||||
self.assertNotIn("failed", result, result)
|
||||
|
||||
def _fold(self, dest: Path, mode: str = "0644") -> None:
|
||||
with patch_module_args({"files": [], "mode": mode, "diff_paths": []}):
|
||||
module = AnsibleModule(
|
||||
argument_spec=_MOD.ARGUMENT_SPEC, supports_check_mode=True
|
||||
)
|
||||
_MOD.write_file(module, str(dest), self.CONTENT, mode)
|
||||
|
||||
@staticmethod
|
||||
def _attributes(path: Path) -> tuple:
|
||||
info = path.stat()
|
||||
return (
|
||||
info.st_uid,
|
||||
info.st_gid,
|
||||
stat.S_IMODE(info.st_mode),
|
||||
path.read_bytes(), # nocheck: cache-read the file under test changes between reads
|
||||
)
|
||||
|
||||
def _assert_parity(self, name: str, *, setgid=False, existing=None) -> None:
|
||||
copy_dir, fold_dir = self._twin_dirs(name, setgid=setgid)
|
||||
for directory in (copy_dir, fold_dir):
|
||||
if existing is not None:
|
||||
content, mode, group = existing
|
||||
target = directory / "body.lua"
|
||||
target.write_text(content, encoding="utf-8")
|
||||
os.chown(target, -1, group)
|
||||
target.chmod(mode)
|
||||
self._copy(copy_dir / "body.lua")
|
||||
self._fold(fold_dir / "body.lua")
|
||||
self.assertEqual(
|
||||
self._attributes(fold_dir / "body.lua"),
|
||||
self._attributes(copy_dir / "body.lua"),
|
||||
)
|
||||
|
||||
def test_a_new_file(self):
|
||||
self._assert_parity("new")
|
||||
|
||||
def test_a_new_file_in_a_setgid_directory(self):
|
||||
self._assert_parity("setgid", setgid=True)
|
||||
|
||||
def test_a_changed_file_keeps_its_group_and_gets_the_mode(self):
|
||||
self._assert_parity("changed", existing=("old\n", 0o600, self.group))
|
||||
|
||||
def test_an_unchanged_file_only_gets_the_mode(self):
|
||||
self._assert_parity("unchanged", existing=(self.CONTENT, 0o600, self.group))
|
||||
|
||||
def test_a_mode_difference_is_detected(self):
|
||||
copy_dir, fold_dir = self._twin_dirs("control", setgid=False)
|
||||
self._copy(copy_dir / "body.lua")
|
||||
self._fold(fold_dir / "body.lua", mode="0600")
|
||||
self.assertNotEqual(
|
||||
self._attributes(fold_dir / "body.lua"),
|
||||
self._attributes(copy_dir / "body.lua"),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,189 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from ansible.parsing.dataloader import DataLoader
|
||||
from ansible.template import Templar, trust_as_template
|
||||
|
||||
from utils.cache.files import PROJECT_ROOT, iter_project_files_with_content, read_text
|
||||
from utils.cache.yaml import load_yaml_any
|
||||
from utils.roles.mapping import (
|
||||
ROLE_FILE_HANDLERS_MAIN,
|
||||
ROLE_FILE_TASKS_MAIN,
|
||||
ROLE_FILE_VARS_MAIN,
|
||||
)
|
||||
|
||||
ROLES = PROJECT_ROOT / "roles"
|
||||
VHOST = ROLES / "sys-stk-front-proxy" / "tasks" / "vhost.yml"
|
||||
INJ_TASKS = ROLES / "sys-front-inj-all" / ROLE_FILE_TASKS_MAIN
|
||||
INJ_STANDALONE = ROLES / "sys-front-inj-all" / "tasks" / "standalone.yml"
|
||||
LUA_PATH = "lookup('nginx', 'files.lua', front_proxy_domain)"
|
||||
LUA_SOURCE = "'roles/sys-front-inj-all/templates/body_filter.lua.j2'"
|
||||
LUA_SCOPE = (
|
||||
"template_vars={'application_id': front_proxy_application_id, "
|
||||
"'domain': front_proxy_domain}"
|
||||
)
|
||||
INCLUDE_KEYS = ("include_role", "ansible.builtin.include_role", "import_role")
|
||||
|
||||
|
||||
def _tasks(path):
|
||||
return load_yaml_any(str(path)) or []
|
||||
|
||||
|
||||
def _named(tasks, prefix):
|
||||
return next(t for t in tasks if str(t.get("name", "")).startswith(prefix))
|
||||
|
||||
|
||||
def _walk(tasks):
|
||||
for task in tasks:
|
||||
yield task
|
||||
for key in ("block", "rescue", "always"):
|
||||
yield from _walk(task.get(key) or [])
|
||||
|
||||
|
||||
def _includes_of(role):
|
||||
found = []
|
||||
for name, content in sorted(
|
||||
iter_project_files_with_content(extensions=(".yml",), exclude_tests=True)
|
||||
):
|
||||
path = Path(name)
|
||||
if not path.is_relative_to(ROLES) or "tasks" not in path.parts:
|
||||
continue
|
||||
if role not in content:
|
||||
continue
|
||||
for task in _walk(_tasks(path)):
|
||||
for key in INCLUDE_KEYS:
|
||||
spec = task.get(key)
|
||||
if isinstance(spec, dict) and spec.get("name") == role:
|
||||
found.append((path.relative_to(PROJECT_ROOT), task))
|
||||
return found
|
||||
|
||||
|
||||
def _flat(text):
|
||||
return " ".join(str(text).split())
|
||||
|
||||
|
||||
class TestEveryCallerDecidesWhoWritesTheLua(unittest.TestCase):
|
||||
def test_only_the_vhost_write_enters_without_the_lua_task(self):
|
||||
entries = {
|
||||
str(path): next(task[key] for key in INCLUDE_KEYS if key in task).get(
|
||||
"tasks_from"
|
||||
)
|
||||
for path, task in _includes_of("sys-front-inj-all")
|
||||
}
|
||||
self.assertEqual(
|
||||
entries,
|
||||
{
|
||||
"roles/sys-stk-front-proxy/tasks/vhost.yml": None,
|
||||
"roles/web-app-matomo/tasks/08_reinject_consumers/vhosts.yml": "standalone.yml",
|
||||
},
|
||||
)
|
||||
|
||||
def test_the_main_entry_writes_no_lua(self):
|
||||
self.assertNotIn("body_filter", read_text(str(INJ_TASKS)))
|
||||
self.assertNotIn("stack_host_template", read_text(str(INJ_TASKS)))
|
||||
|
||||
def test_the_standalone_entry_builds_the_injection_then_writes_the_lua(self):
|
||||
build, lua = _tasks(INJ_STANDALONE)
|
||||
self.assertEqual(build["ansible.builtin.include_tasks"], "main.yml")
|
||||
self.assertEqual(lua["stack_host_template"]["src"], "body_filter.lua.j2")
|
||||
self.assertEqual(lua["notify"], "reload-openresty")
|
||||
self.assertNotIn("when", lua)
|
||||
|
||||
|
||||
class TestTheVhostWriteCarriesTheLua(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tasks = _tasks(VHOST)
|
||||
cls.deploy = _named(cls.tasks, "🚀 Deploy NGINX config")
|
||||
cls.files = _flat(cls.deploy["stack_host_contents"]["files"])
|
||||
|
||||
def test_the_lua_is_the_first_file_and_skipped_off_the_stack_host(self):
|
||||
lua = self.files.index(LUA_PATH)
|
||||
self.assertLess(lua, self.files.index("protocol='http'"))
|
||||
self.assertLess(lua, self.files.index("front_proxy_domain_conf_dst_resolved"))
|
||||
self.assertIn(
|
||||
f"lookup('template_shallow', lookup('path_absolute', {LUA_SOURCE}), "
|
||||
f"{LUA_SCOPE})}}] if IS_STACK_HOST | bool else [])",
|
||||
self.files,
|
||||
)
|
||||
|
||||
def test_the_lua_scope_matches_the_include_params(self):
|
||||
include = _named(self.tasks, "📌 CSP/INJ")["vars"]
|
||||
self.assertEqual(
|
||||
include,
|
||||
{
|
||||
"application_id": "{{ front_proxy_application_id }}",
|
||||
"domain": "{{ front_proxy_domain }}",
|
||||
},
|
||||
)
|
||||
self.assertIn(LUA_SCOPE, self.files)
|
||||
|
||||
def test_the_lua_renders_from_the_template_the_action_rendered(self):
|
||||
inj = _named(_tasks(INJ_STANDALONE), "📄 Render body_filter Lua")
|
||||
self.assertIn(
|
||||
f"'{'roles/sys-front-inj-all/templates/'}{inj['stack_host_template']['src']}'",
|
||||
self.files,
|
||||
)
|
||||
self.assertEqual(
|
||||
_flat(inj["stack_host_template"]["dest"]),
|
||||
"{{ " + LUA_PATH.replace("front_proxy_domain", "domain") + " }}",
|
||||
)
|
||||
|
||||
def test_mode_handler_and_diff_match_the_action(self):
|
||||
inj = _named(_tasks(INJ_STANDALONE), "📄 Render body_filter Lua")
|
||||
args = self.deploy["stack_host_contents"]
|
||||
self.assertEqual(args["mode"], inj["stack_host_template"]["mode"])
|
||||
self.assertEqual(self.deploy["notify"], inj["notify"])
|
||||
self.assertEqual(args["diff_paths"], ["{{ " + LUA_PATH + " }}"])
|
||||
|
||||
def test_injection_runs_before_the_write_and_the_recheck_after_the_reload(self):
|
||||
names = [str(t.get("name", "")) for t in self.tasks]
|
||||
order = [
|
||||
next(i for i, n in enumerate(names) if n.startswith(prefix))
|
||||
for prefix in (
|
||||
"📌 CSP/INJ",
|
||||
"🚀 Deploy NGINX config",
|
||||
"🔄 Restart Webserver",
|
||||
"🔁 Re-check",
|
||||
)
|
||||
]
|
||||
self.assertEqual(order, sorted(order))
|
||||
recheck = _named(self.tasks, "🔁 Re-check")
|
||||
probe, redeploy, flush = recheck["block"]
|
||||
self.assertIn("uri_probe", probe)
|
||||
self.assertEqual(redeploy["notify"], "restart-openresty")
|
||||
self.assertEqual(flush["ansible.builtin.meta"], "flush_handlers")
|
||||
self.assertEqual(recheck["when"], "front_proxy_vhost_changed")
|
||||
|
||||
|
||||
class TestALuaOnlyChangeSkipsTheVhostRecheck(unittest.TestCase):
|
||||
def _vhost_changed(self, changed, paths):
|
||||
expression = load_yaml_any(
|
||||
str(ROLES / "sys-stk-front-proxy" / ROLE_FILE_VARS_MAIN)
|
||||
)["front_proxy_vhost_changed"]
|
||||
variables = {
|
||||
"nginx_conf": {"changed": changed, "changed_paths": paths},
|
||||
"front_proxy_domain_conf_dst_resolved": "/etc/nginx/conf.d/x.conf",
|
||||
}
|
||||
return Templar(loader=DataLoader(), variables=variables).template(
|
||||
trust_as_template(expression)
|
||||
)
|
||||
|
||||
def test_only_a_vhost_change_triggers_the_recheck(self):
|
||||
lua = "/etc/nginx/conf.d/lua/x.lua"
|
||||
self.assertFalse(self._vhost_changed(True, [lua]))
|
||||
self.assertTrue(self._vhost_changed(True, [lua, "/etc/nginx/conf.d/x.conf"]))
|
||||
self.assertFalse(self._vhost_changed(False, []))
|
||||
|
||||
def test_the_handler_both_writers_notify_exists(self):
|
||||
handlers = read_text(str(ROLES / "svc-prx-openresty" / ROLE_FILE_HANDLERS_MAIN))
|
||||
self.assertRegex(
|
||||
handlers, re.compile(r"^\s*listen: reload-openresty$", re.MULTILINE)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user