mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-08 11:46:35 +00:00
Merge branch 'feature/updater-lockdown-probe' into feature/updater
This commit is contained in:
44 files changed
+1134
-94
No files matched your search
@@ -101,8 +101,6 @@ def placed_roles(roles_dir: Path) -> list[Path]:
|
||||
|
||||
|
||||
def _fields(role: Path) -> dict[str, str]:
|
||||
# Exception: O_NONBLOCK keeps a FIFO planted under the marker name from
|
||||
# blocking the open; the fstat below refuses it.
|
||||
try:
|
||||
descriptor = os.open(
|
||||
role / MARKER_FILE, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK
|
||||
|
||||
@@ -156,10 +156,6 @@ def _check_local(
|
||||
|
||||
|
||||
def _git(cwd: str | None, arguments: list[str], failure: str) -> bytes:
|
||||
# Exception: a git hook exports GIT_DIR, GIT_INDEX_FILE and friends, which
|
||||
# would point the scratch repository at the core checkout; the askpass
|
||||
# helpers and the detached session keep a fetch that needs credentials
|
||||
# from prompting.
|
||||
env = {key: value for key, value in os.environ.items() if key not in _DROPPED_ENV}
|
||||
env.update(GIT_TERMINAL_PROMPT="0", SSH_ASKPASS_REQUIRE="never")
|
||||
result = subprocess.run(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# nocheck: comments-valid explanatory WHY-comments in this SPOT lookup predate the stricter lint
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -11,6 +12,7 @@ from ansible.plugins.lookup import LookupBase
|
||||
from plugins.lookup.applications import LookupModule as ApplicationsLookup
|
||||
from plugins.lookup.domain import LookupModule as DomainLookup
|
||||
from plugins.lookup.users import LookupModule as UsersLookup
|
||||
from utils.cache.carrier import APPLICATIONS_RENDERED_FACT
|
||||
from utils.domains.primary_domain import get_domain
|
||||
from utils.mail.provider import deployed_roles, resolve_active_provider
|
||||
from utils.roles.entity.name import get_entity_name
|
||||
@@ -88,6 +90,19 @@ def _render(
|
||||
return value
|
||||
|
||||
|
||||
def _provider_carrier(variables: dict[str, Any], host: str) -> Mapping[str, Any] | None:
|
||||
hostvars = variables.get("hostvars")
|
||||
raw_get = getattr(hostvars, "raw_get", None)
|
||||
try:
|
||||
host_vars = raw_get(host) if callable(raw_get) else hostvars[host]
|
||||
except (KeyError, TypeError):
|
||||
return None
|
||||
if not isinstance(host_vars, Mapping):
|
||||
return None
|
||||
carrier = host_vars.get(APPLICATIONS_RENDERED_FACT)
|
||||
return carrier if isinstance(carrier, Mapping) else None
|
||||
|
||||
|
||||
class LookupModule(LookupBase):
|
||||
def run(
|
||||
self,
|
||||
@@ -313,6 +328,9 @@ class LookupModule(LookupBase):
|
||||
provider_groups = fallback
|
||||
provider_view = dict(variables)
|
||||
provider_view["group_names"] = list(provider_groups or fallback)
|
||||
carrier = _provider_carrier(variables, provider_hosts[0])
|
||||
if carrier is not None:
|
||||
provider_view[APPLICATIONS_RENDERED_FACT] = carrier
|
||||
forwarded = {
|
||||
k: v for k, v in getattr(self, "_kwargs", {}).items() if k == "roles_dir"
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ ollama:
|
||||
backup:
|
||||
no_stop_required: true
|
||||
image: ollama/ollama
|
||||
version: 0.40.12
|
||||
version: 0.40.13
|
||||
name: ollama
|
||||
min_storage: 3473MB
|
||||
cpus: "4.0"
|
||||
|
||||
@@ -25,12 +25,13 @@ Env (rendered into test.env from templates/test.env.j2):
|
||||
ONION_HOST the node onion host
|
||||
ONION_PORTS space-separated forwarded ports; empty means there is
|
||||
nothing to check
|
||||
ONION_PORT_TIMEOUT seconds per connect attempt. 60 for the same reason the
|
||||
domain probe uses it: Tor's SocksTimeout default is 120s
|
||||
and abandoning earlier is destructive, because the next
|
||||
attempt opens a fresh SOCKS connection and throws away
|
||||
the descriptor fetches and rendezvous retries the last
|
||||
one was still working through
|
||||
ONION_PORT_TIMEOUT seconds per connect attempt, above Tor's SocksTimeout
|
||||
(120s by default; torrc does not set it). Every attempt
|
||||
builds a fresh rendezvous, which Tor keeps working on
|
||||
until SocksTimeout and then ends itself with reply 0x06
|
||||
and a "Giving up" notice in its log. An attempt abandoned
|
||||
earlier throws away a rendezvous Tor may still complete
|
||||
and leaves Tor's log without a trace of the failure
|
||||
ONION_PORT_RETRIES passes over the failing ports. Below the domain probe's
|
||||
RETRIES because this phase multiplies by the port count,
|
||||
and meta/tests.yml caps the whole CLI test
|
||||
@@ -148,7 +149,7 @@ def main() -> int:
|
||||
retries = int(_required("ONION_PORT_RETRIES", "passes over the failing ports"))
|
||||
sleep_seconds = float(_required("SLEEP_SECONDS", "wait between passes"))
|
||||
timeout = float(
|
||||
_required("ONION_PORT_TIMEOUT", "seconds per connect, flavor-dependent")
|
||||
_required("ONION_PORT_TIMEOUT", "seconds per connect, above Tor's SocksTimeout")
|
||||
)
|
||||
ports = [int(entry) for entry in os.environ.get("ONION_PORTS", "").split()]
|
||||
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
---
|
||||
cli:
|
||||
timeout: 7200
|
||||
timeout: 9000
|
||||
@@ -80,7 +80,6 @@
|
||||
dest: "{{ TOR_ENTRYPOINT_HOST }}"
|
||||
mode: "0755"
|
||||
become: true
|
||||
register: _tor_entrypoint
|
||||
|
||||
- name: "🩺 Stage the Tor container healthcheck into the build context"
|
||||
stack_host_copy:
|
||||
@@ -88,7 +87,28 @@
|
||||
dest: "{{ TOR_HEALTHCHECK_HOST }}"
|
||||
mode: "0755"
|
||||
become: true
|
||||
register: _tor_healthcheck
|
||||
|
||||
- name: "🧩 Load the container CLI the Tor run probe calls"
|
||||
include_role:
|
||||
name: sys-svc-container
|
||||
public: false
|
||||
when: run_once_sys_svc_container is not defined
|
||||
|
||||
- name: "🔍 Check whether Tor already runs from an earlier deploy"
|
||||
ansible.builtin.command: >-
|
||||
{% if DEPLOYMENT_MODE == 'swarm' %}
|
||||
container service ps -q --filter desired-state=running
|
||||
{{ lookup('container_service', application_id, 'tor') }}
|
||||
{% else %}
|
||||
container ps -q --filter status=running --filter name=^{{ TOR_CONTAINER }}$
|
||||
{% endif %}
|
||||
register: _tor_running_before
|
||||
changed_when: false
|
||||
failed_when:
|
||||
- _tor_running_before.rc != 0
|
||||
- "'no such service' not in (_tor_running_before.stderr | lower)"
|
||||
become: true
|
||||
when: IS_STACK_HOST | bool
|
||||
|
||||
- name: "🧩 For '{{ application_id }}': Load sys-svc-compose"
|
||||
include_role:
|
||||
|
||||
@@ -2,7 +2,7 @@ NGINX_SERVERS_DIR={{ lookup('nginx', 'directories.configuration.servers') }}
|
||||
ONION_HOST={{ lookup('config', 'svc-net-tor', 'services.tor.node') }}
|
||||
ONION_PORTS={{ lookup('tor_ports') | map(attribute='onion_port') | unique | sort | join(' ') | dotenv_quote }}
|
||||
ONION_PORT_RETRIES=2
|
||||
ONION_PORT_TIMEOUT=60
|
||||
ONION_PORT_TIMEOUT=130
|
||||
RETRIES=5
|
||||
SLEEP_SECONDS=10
|
||||
TOR_CONTAINER={{ lookup('container_address', application_id, 'tor') }}
|
||||
|
||||
@@ -60,7 +60,6 @@ TOR_HEALTHCHECK_HOST: "{{ [ TOR_INSTANCE_DIR, 'healthcheck.sh' ] | path_join }}"
|
||||
TOR_ENTRYPOINT_PATH: "/usr/local/bin/tor-entrypoint.sh"
|
||||
TOR_HEALTHCHECK_PATH: "/usr/local/bin/tor-healthcheck.sh"
|
||||
TOR_RESTART_REQUIRED: >-
|
||||
{{ (_tor_hs_keys is changed)
|
||||
or (_tor_torrc is changed)
|
||||
or (_tor_entrypoint is changed)
|
||||
or (_tor_healthcheck is changed) }}
|
||||
{{ ((_tor_hs_keys is changed) or (_tor_torrc is changed))
|
||||
and (_tor_running_before is not skipped)
|
||||
and (_tor_running_before.stdout | trim | length > 0) }}
|
||||
@@ -45,7 +45,8 @@
|
||||
path: "{{ SVC_STORAGE_NFS_CLIENT_PROBE_DIR }}"
|
||||
src: "{{ STORAGE_NFS_CLIENT_SRC }}"
|
||||
fstype: "{{ STORAGE_NFS_FSTYPE }}"
|
||||
opts: "{{ STORAGE_NFS_MOUNT_OPTS }},_netdev,noauto"
|
||||
opts: "{{ STORAGE_NFS_MOUNT_OPTS }},_netdev"
|
||||
boot: false
|
||||
state: ephemeral
|
||||
register: svc_storage_nfs_client_probe
|
||||
failed_when:
|
||||
|
||||
@@ -53,7 +53,8 @@ Solid `1:1` edges are fixed relationships; dashed `0..1` edges are conditional (
|
||||
to every routing-mesh port no application declares under `ports.public`.
|
||||
Loopback, the Docker bridges and the swarm nodes' own addresses stay
|
||||
admitted. The table is loaded before a node joins and reloaded on every node
|
||||
after the join and after each stack deploy.
|
||||
after the join and after each stack deploy that changes the published ports
|
||||
or the swarm nodes.
|
||||
|
||||
## Quick Setup
|
||||
|
||||
|
||||
@@ -11,6 +11,11 @@
|
||||
file: "{{ lookup('path_absolute', 'roles/svc-net-firewall/vars/main.yml') }}"
|
||||
name: swarm_ingress_firewall
|
||||
|
||||
- name: "🧮 Swarm ingress guard | Fingerprint the bound the swarm nodes need"
|
||||
ansible.builtin.set_fact:
|
||||
swarm_ingress_guard_digest: "{{ [SWARM_INGRESS_GUARDED, SWARM_INGRESS_PEERS] | to_json | hash('sha256') }}"
|
||||
run_once: true
|
||||
|
||||
- name: "🛡️ Swarm ingress guard | Reload the bound on every swarm node"
|
||||
ansible.builtin.include_tasks:
|
||||
file: "{{ lookup('path_absolute', 'roles/svc-net-firewall/tasks/02_fragment.yml') }}"
|
||||
@@ -28,3 +33,14 @@
|
||||
loop_control:
|
||||
loop_var: swarm_ingress_guard_node
|
||||
run_once: true
|
||||
when: (hostvars[swarm_ingress_guard_node].swarm_ingress_guard_loaded | default('')) != swarm_ingress_guard_digest
|
||||
|
||||
- name: "🧾 Swarm ingress guard | Record the bound every swarm node now carries"
|
||||
ansible.builtin.set_fact:
|
||||
swarm_ingress_guard_loaded: "{{ swarm_ingress_guard_digest }}"
|
||||
delegate_to: "{{ swarm_ingress_guard_node }}"
|
||||
delegate_facts: true
|
||||
loop: "{{ groups['svc-swarm-node'] }}"
|
||||
loop_control:
|
||||
loop_var: swarm_ingress_guard_node
|
||||
run_once: true
|
||||
@@ -24,14 +24,24 @@
|
||||
ansible.builtin.set_fact:
|
||||
service_loader_post_run: "{{ (service_loader_post_run | default([])) + list_or_shoot_roles }}"
|
||||
changed_when: false
|
||||
run_once: false
|
||||
when: (service_loader_preloading | default(false) | bool)
|
||||
|
||||
- name: "🎯 Shoot the post-load role directly (no preload context active)"
|
||||
ansible.builtin.include_role:
|
||||
name: "{{ list_or_shoot_roles | first }}"
|
||||
vars:
|
||||
application_id: "{{ list_or_shoot_roles | first }}"
|
||||
when:
|
||||
- not (service_loader_preloading | default(false) | bool)
|
||||
- list_or_shoot_roles | length == 1
|
||||
|
||||
- name: "🎯 Shoot post-load roles directly (no preload context active)"
|
||||
ansible.builtin.include_role:
|
||||
name: "{{ list_or_shoot_role }}"
|
||||
vars:
|
||||
application_id: "{{ list_or_shoot_role }}"
|
||||
loop: "{{ list_or_shoot_roles }}"
|
||||
loop: "{{ list_or_shoot_roles if list_or_shoot_roles | length > 1 else [] }}"
|
||||
loop_control:
|
||||
loop_var: list_or_shoot_role
|
||||
when: not (service_loader_preloading | default(false) | bool)
|
||||
@@ -55,6 +55,11 @@
|
||||
loop_var: service_item
|
||||
label: "{{ service_item.id }}"
|
||||
|
||||
- name: "📂 Close the preload context before the post-load flush"
|
||||
ansible.builtin.set_fact:
|
||||
service_loader_preloading: false
|
||||
changed_when: false
|
||||
|
||||
- name: "▶️ Run service-loader post-load roles"
|
||||
ansible.builtin.include_role:
|
||||
name: "{{ post_run_role }}"
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
---
|
||||
- name: "🔍 Probe the vhosts this play queued"
|
||||
when: front_proxy_deferred_probes | length > 0
|
||||
when:
|
||||
- front_proxy_deferred_probes | length > 0
|
||||
- inventory_hostname == groups['svc-prx-openresty'][0]
|
||||
block:
|
||||
- name: "🔍 Check if the queued vhosts are reachable"
|
||||
uri_probe_batch:
|
||||
|
||||
@@ -55,6 +55,7 @@
|
||||
url: "{{ lookup('tls', front_proxy_domain, 'url.base') }}"
|
||||
status_codes: "{{ lookup('applications') | accepted_status_codes(front_proxy_application_id, lookup('tls', front_proxy_domain, 'domain')) }}"
|
||||
changed_when: false
|
||||
when: IS_STACK_HOST | bool
|
||||
|
||||
- name: 📂 Include dependencies
|
||||
include_role:
|
||||
|
||||
@@ -2,15 +2,12 @@
|
||||
- include_tasks: 00_core.yml
|
||||
when: run_once_sys_svc_compose is not defined
|
||||
|
||||
- name: "🧭 Route docker cleanup/health deps through service-loader"
|
||||
- name: "🧭 Route the compose hard repair through service-loader"
|
||||
ansible.builtin.include_role:
|
||||
name: sys-service-loader
|
||||
tasks_from: list_or_shoot.yml
|
||||
vars:
|
||||
list_or_shoot_roles:
|
||||
- sys-ctl-cln-docker
|
||||
- sys-ctl-hlth-container
|
||||
- sys-ctl-hlth-volumes
|
||||
- sys-ctl-rpr-container-hard
|
||||
|
||||
- name: "🧭 Route svc-bkp-volume-2-local through service-loader (when deployed)"
|
||||
|
||||
@@ -94,4 +94,14 @@
|
||||
include_tasks: "03_reset.yml"
|
||||
when: MODE_RESET | bool
|
||||
|
||||
- name: "🧭 Route docker cleanup/health deps through service-loader"
|
||||
ansible.builtin.include_role:
|
||||
name: sys-service-loader
|
||||
tasks_from: list_or_shoot.yml
|
||||
vars:
|
||||
list_or_shoot_roles:
|
||||
- sys-ctl-cln-docker
|
||||
- sys-ctl-hlth-container
|
||||
- sys-ctl-hlth-volumes
|
||||
|
||||
- include_tasks: utils/once/flag.yml
|
||||
@@ -1,4 +1,7 @@
|
||||
---
|
||||
# REQUIRED:
|
||||
# sys_svc_container_prepull_extra # image references to pull besides the app images of the deploy, e.g. the E2E runner
|
||||
|
||||
- name: "📥 Pre-pull the images of this deploy in the background"
|
||||
when:
|
||||
- not (STORAGE_CONSTRAINED | bool)
|
||||
@@ -8,10 +11,9 @@
|
||||
- name: "🧮 Derive the images the deploy of the host runs"
|
||||
ansible.builtin.set_fact:
|
||||
sys_svc_container_prepull_plan: >-
|
||||
{{
|
||||
lookup('applications')
|
||||
| update_pull_images(lookup('application_closure', group_names | intersect(lookup('deployment').running)))
|
||||
}}
|
||||
{%- set plan = lookup('applications')
|
||||
| update_pull_images(lookup('application_closure', group_names | intersect(lookup('deployment').running))) -%}
|
||||
{{ plan | combine({'pull': plan.pull + (sys_svc_container_prepull_extra | reject('in', plan.pull) | list)}) }}
|
||||
|
||||
- name: "📥 Pull the images the host does not hold yet"
|
||||
ansible.builtin.include_tasks: "{{ role_path }}/tasks/utils/prepull/pull.yml"
|
||||
|
||||
@@ -42,12 +42,14 @@
|
||||
block:
|
||||
- name: 🚀 Run CLI test for each detected role
|
||||
ansible.builtin.include_tasks: run_one.yml
|
||||
vars:
|
||||
application_id: "{{ cli_role_id }}"
|
||||
loop: "{{ _cli_apps }}"
|
||||
loop_control:
|
||||
loop_var: application_id # nocheck: loop-var-shadow the item IS the role id, so run_one reads the same value the tested role declares
|
||||
label: "{{ application_id }}"
|
||||
loop_var: cli_role_id
|
||||
label: "{{ cli_role_id }}"
|
||||
when: >-
|
||||
not (lookup('role_tests', application_id, 'cli.controller_only', default=false) | bool)
|
||||
not (lookup('role_tests', cli_role_id, 'cli.controller_only', default=false) | bool)
|
||||
or (ansible_connection | default('')) == 'local'
|
||||
|
||||
- name: ✅ Fail if there were CLI test failures
|
||||
|
||||
@@ -16,10 +16,12 @@
|
||||
|
||||
- name: ▶️ Run Playwright for each detected role
|
||||
ansible.builtin.include_tasks: 02_run_one.yml
|
||||
vars:
|
||||
application_id: "{{ playwright_role_id }}"
|
||||
loop: "{{ TEST_E2E_PLAYWRIGHT_APPS }}"
|
||||
loop_control:
|
||||
loop_var: application_id # nocheck: loop-var-shadow the item IS the role id, so 02_run_one reads the same value the tested role's own vars declare
|
||||
label: "{{ application_id }}"
|
||||
loop_var: playwright_role_id
|
||||
label: "{{ playwright_role_id }}"
|
||||
|
||||
- name: 🔁 Stash first-pass failures for one flake retry
|
||||
ansible.builtin.set_fact:
|
||||
@@ -29,10 +31,12 @@
|
||||
|
||||
- name: 🔁 Retry failed roles once on rotated Tor circuits
|
||||
ansible.builtin.include_tasks: 02_run_one.yml
|
||||
vars:
|
||||
application_id: "{{ playwright_role_id }}"
|
||||
loop: "{{ _playwright_retry_roles | default([]) }}"
|
||||
loop_control:
|
||||
loop_var: application_id # nocheck: loop-var-shadow same loop body, same identity between item and role id
|
||||
label: "{{ application_id }} (retry)"
|
||||
loop_var: playwright_role_id
|
||||
label: "{{ playwright_role_id }} (retry)"
|
||||
|
||||
- name: 🛑 Fail if there were Playwright failures
|
||||
ansible.builtin.fail:
|
||||
|
||||
@@ -234,10 +234,6 @@ async function pdsAdminUpdatePassword(did, newPassword) {
|
||||
|
||||
// --- High-level orchestration --------------------------------------
|
||||
|
||||
// Exception: the account map lives in the broker's own volume, never in a
|
||||
// Keycloak user attribute; a bluesky_* attribute write under LDAP federation
|
||||
// in WRITABLE mode triggers a schema-violating sync push that breaks later
|
||||
// OIDC logins of that user.
|
||||
const store = openStore(CONFIG.storePath);
|
||||
|
||||
class HandleChoiceRequired extends Error {}
|
||||
|
||||
@@ -62,7 +62,7 @@ web:
|
||||
view:
|
||||
enabled: false
|
||||
repository: https://github.com/bluesky-social/social-app.git
|
||||
ref: "1.136.0"
|
||||
ref: "1.137.0"
|
||||
ports:
|
||||
local:
|
||||
http: 8051
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# To-dos
|
||||
|
||||
- Move off PHP 7.4. The pinned `nfrastack/fusiondirectory:1.5` image builds on `ghcr.io/nfrastack/container-nginx-php-fpm` with PHP 7.4 on Alpine 3.15, which is end of life and gets no fix for CVE-2026-109618 and CVE-2026-109619.
|
||||
- Mount the shared PHP hardening ini (`FILE_PHP_HARDENING_INI`) once the image's scan dir is verified. The nfrastack init scripts manage `conf.d` at runtime, so a read-only mount there has to be tested before it ships.
|
||||
- No FusionDirectory release runs on a supported PHP yet: upstream certifies 1.4 to 1.6 for PHP 7.4 only, `nfrastack/container-fusiondirectory` builds its image only on PHP 7.4 and Alpine 3.15, and the PHP 8 port lives on upstream's unreleased `php8` branch. Until then the shared hardening ini mounted at `/etc/php7/conf.d/zz-infinito-hardening.ini` carries the mitigations for both CVEs. Options: rebuild the image on upstream's first `php8` release, or build `nfrastack/container-fusiondirectory` against a pinned `php8` commit on `container-nginx-php-fpm` 8.4 before that release.
|
||||
@@ -1,4 +1,11 @@
|
||||
---
|
||||
php_hardening:
|
||||
type: config
|
||||
source: "{{ FILE_PHP_HARDENING_INI }}"
|
||||
mode: "0444"
|
||||
mounts:
|
||||
- service: fusiondirectory
|
||||
target: /etc/php7/conf.d/zz-infinito-hardening.ini
|
||||
# nocheck: embedded-state-on-nfs generated asset cache; the authoritative state lives in LDAP
|
||||
data:
|
||||
type: volume
|
||||
|
||||
@@ -11,6 +11,6 @@ server {
|
||||
|
||||
{% include 'roles/sys-svc-proxy/templates/location/upload.conf.j2' %}
|
||||
|
||||
{% set container_name = MATRIX_SYNAPSE_NAME if matrix_flavor == 'compose' else MATRIX_MDAD_CONTAINER %}
|
||||
{% set container_name = MATRIX_SYNAPSE_SERVICE if matrix_flavor == 'compose' else lookup('container_service', application_id, MATRIX_MDAD_SERVICE_KEY) %}
|
||||
{% include 'roles/web-app-prometheus/templates/nginx/locations.conf.j2' %}
|
||||
}
|
||||
@@ -32,6 +32,10 @@
|
||||
end
|
||||
|
||||
metric_stack_up:set(up, {ngx.var.host, ngx.var.app_id})
|
||||
if metric_docker_health_entries then
|
||||
local health = ngx.shared.health_containers
|
||||
metric_docker_health_entries:set(health and #health:get_keys(0) or 0)
|
||||
end
|
||||
prometheus:collect()
|
||||
}
|
||||
}
|
||||
@@ -89,6 +89,41 @@ test("metricz endpoint is accessible and returns prometheus text format", async
|
||||
)
|
||||
.toBe(true);
|
||||
}
|
||||
|
||||
const metriczHost = new URL(metriczUrl).hostname.toLowerCase();
|
||||
const targetIds = new Set(prometheusTargetRoles.map((target) => target.id));
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const scrape = await apiGetOnion(request, metriczUrl, { timeout: resolveTimeout(30_000) });
|
||||
const text = await scrape.text();
|
||||
const entries = text.match(/^nginx_docker_health_entries (\S+)$/m);
|
||||
const stackUp = [...text.matchAll(/^nginx_stack_up\{host="([^"]*)",app="([^"]*)"\} (\S+)$/gm)];
|
||||
const failures = [];
|
||||
if (!entries) {
|
||||
failures.push("nginx_docker_health_entries is missing");
|
||||
} else if (!(Number(entries[1]) > 0)) {
|
||||
failures.push(`nginx_docker_health_entries is ${entries[1]}`);
|
||||
}
|
||||
if (!stackUp.some(([, host]) => host === metriczHost)) {
|
||||
failures.push(`nginx_stack_up{host="${metriczHost}"} is missing`);
|
||||
}
|
||||
for (const [, host, app, value] of stackUp) {
|
||||
if ((host === metriczHost || targetIds.has(app)) && value !== "1") {
|
||||
failures.push(`nginx_stack_up{host="${host}",app="${app}"} is ${value}`);
|
||||
}
|
||||
}
|
||||
return failures;
|
||||
},
|
||||
{
|
||||
timeout: resolveTimeout(60_000),
|
||||
message:
|
||||
"/metricz must show a populated docker-health dict (nginx_docker_health_entries > 0) " +
|
||||
"and nginx_stack_up 1 for the scraped vhost and every consumer role that has a series. " +
|
||||
"An entries count of 0 means the OpenResty privileged agent cannot read /var/run/docker.sock.",
|
||||
},
|
||||
)
|
||||
.toEqual([]);
|
||||
});
|
||||
|
||||
// Scenario II: direct goto Prometheus → SSO login (as admin) → verify Prometheus UI → logout
|
||||
|
||||
@@ -71,7 +71,7 @@
|
||||
stack_host_template:
|
||||
src: "roles/web-app-prometheus/templates/nginx/prometheus.conf.j2"
|
||||
dest: "{{ [ lookup('nginx','directories.configuration.global'), 'prometheus.conf' ] | path_join }}"
|
||||
notify: restart-openresty
|
||||
notify: reload-openresty
|
||||
|
||||
- name: "🧱 Rebuild the MCP adapter of {{ application_id }}"
|
||||
ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/svc-ai-mcp-adapter/tasks/rebuild.yml') }}"
|
||||
|
||||
@@ -12,7 +12,9 @@
|
||||
service registry.
|
||||
|
||||
$app_id — Ansible role ID attached as Lua metrics label.
|
||||
$container_name — Docker container name for /healthz/live polling.
|
||||
$container_name — health_containers key for /healthz/live polling: the
|
||||
container name in compose, the swarm service name
|
||||
(lookup('container_service')) in swarm.
|
||||
Callers may set {% set container_name = "..." %} before
|
||||
this include to override the default (derived from the
|
||||
canonical service key via get_entity_name).
|
||||
@@ -27,7 +29,7 @@
|
||||
set $container_name "{{ container_name }}";
|
||||
{% else %}
|
||||
{% set prometheus_service_key = application_id | get_entity_name %}
|
||||
set $container_name "{{ lookup('config', application_id, 'services.' + prometheus_service_key + '.name', '') }}";
|
||||
set $container_name "{{ lookup('container_service', application_id, prometheus_service_key) if lookup('config', application_id, 'services.' + prometheus_service_key + '.name', '') else '' }}";
|
||||
{% endif %}
|
||||
{% include 'roles/web-app-prometheus/templates/nginx/healthz.conf.j2' %}
|
||||
{% include 'roles/web-app-prometheus/templates/nginx/location.conf.j2' %}
|
||||
|
||||
@@ -4,6 +4,13 @@
|
||||
|
||||
lua_shared_dict health_containers 1m;
|
||||
|
||||
init_by_lua_block {
|
||||
local ok, err = require("ngx.process").enable_privileged_agent()
|
||||
if not ok then
|
||||
ngx.log(ngx.ERR, "docker-health: cannot enable the privileged agent: ", err)
|
||||
end
|
||||
}
|
||||
|
||||
init_worker_by_lua_block {
|
||||
-- lua-resty-prometheus (master) requires init() in init_worker_by_lua_block,
|
||||
-- not init_by_lua_block, so that each worker initialises its own state.
|
||||
@@ -34,18 +41,17 @@
|
||||
"Overall stack availability per vhost (1=up, 0=down)",
|
||||
{"host", "app"}
|
||||
)
|
||||
metric_docker_health_entries = prometheus:gauge(
|
||||
"nginx_docker_health_entries",
|
||||
"Unexpired health_containers entries written by the docker-health poller"
|
||||
)
|
||||
metric_tls_handshakes = prometheus:counter(
|
||||
"nginx_tls_handshakes_total",
|
||||
"Total TLS handshakes by protocol",
|
||||
{"host", "app", "protocol"}
|
||||
)
|
||||
|
||||
-- Docker health polling — worker 0 only to avoid duplicate socket traffic.
|
||||
-- Queries GET /v1.41/containers/json via the Docker Unix socket, then
|
||||
-- stores each container's liveness in health_containers. /healthz/live
|
||||
-- reads the cache synchronously so Docker state is checked on every probe
|
||||
-- without adding socket-connect latency to the request path.
|
||||
if ngx.worker.id and ngx.worker.id() == 0 then
|
||||
if require("ngx.process").type() == "privileged agent" then
|
||||
local function poll_docker_health(premature)
|
||||
if premature then return end
|
||||
local cjson = require("cjson.safe")
|
||||
@@ -81,6 +87,7 @@
|
||||
local containers = cjson.decode(body)
|
||||
if type(containers) ~= "table" then return end
|
||||
|
||||
local services = {}
|
||||
for _, c in ipairs(containers) do
|
||||
local names = c.Names
|
||||
if type(names) == "table" and names[1] then
|
||||
@@ -95,8 +102,17 @@
|
||||
local is_up = (state == "running") and (health_tag ~= "unhealthy")
|
||||
-- TTL 60 s — entries expire if the container disappears between polls
|
||||
health:set(name, is_up and "up" or "down", 60)
|
||||
local labels = c.Labels
|
||||
local service = type(labels) == "table" and labels["com.docker.swarm.service.name"]
|
||||
if type(service) == "string" and service ~= "" then
|
||||
services[service] = services[service] or is_up
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
for service, up in pairs(services) do
|
||||
health:set(service, up and "up" or "down", 60)
|
||||
end
|
||||
end
|
||||
|
||||
-- Seed immediately so the dict is populated before the first request
|
||||
|
||||
@@ -33,12 +33,14 @@ test("administrator: stalwart recovery admin native login and logout", async ({
|
||||
.first()
|
||||
.click();
|
||||
|
||||
// Stalwart asks consent for every registered OAuth client before it shows the password form.
|
||||
// Stalwart v1.0.0 asks consent for the OAuth client before the password form; v1.0.1 shows the password form directly.
|
||||
const allow = page.getByRole("button", { name: /^allow$/i });
|
||||
await allow.waitFor({ state: "visible", timeout: resolveTimeout(30_000) });
|
||||
await allow.click();
|
||||
|
||||
const passwordField = page.locator("input[type='password'], input[name='password']").first();
|
||||
await allow.or(passwordField).first().waitFor({ state: "visible", timeout: resolveTimeout(30_000) });
|
||||
if (await allow.isVisible()) {
|
||||
await allow.click();
|
||||
}
|
||||
|
||||
await passwordField.waitFor({ state: "visible", timeout: resolveTimeout(30_000) });
|
||||
await passwordField.fill(stalwartAdminPassword);
|
||||
await page
|
||||
@@ -51,9 +53,15 @@ test("administrator: stalwart recovery admin native login and logout", async ({
|
||||
page.getByText(/invalid username or password/i)
|
||||
).toBeHidden({ timeout: resolveTimeout(15_000) });
|
||||
|
||||
await expect(
|
||||
page.locator("nav, .sidebar, [class*='menu'], h1, h2").filter({ hasText: /dashboard|domains|account|settings|directory/i }).first()
|
||||
).toBeVisible({ timeout: resolveTimeout(30_000) });
|
||||
const dashboard = page
|
||||
.locator("nav, .sidebar, [class*='menu'], h1, h2")
|
||||
.filter({ hasText: /dashboard|domains|account|settings|directory/i })
|
||||
.first();
|
||||
await allow.or(dashboard).first().waitFor({ state: "visible", timeout: resolveTimeout(30_000) });
|
||||
if (await allow.isVisible()) {
|
||||
await allow.click();
|
||||
}
|
||||
await expect(dashboard).toBeVisible({ timeout: resolveTimeout(30_000) });
|
||||
|
||||
if (oidcIssuerUrl) {
|
||||
expect(page.url()).not.toContain(oidcIssuerUrl);
|
||||
|
||||
@@ -2,15 +2,23 @@
|
||||
# Reusable management JMAP call: in `stalwart_jmap_body` (JSON), out `stalwart_jmap`.
|
||||
# Param: stalwart_jmap_body - JSON request; a batch carries one method call per object owner, its call id naming that owner in a refusal.
|
||||
# Param: stalwart_jmap_tolerate_rejected - true when the caller reads a set-level rejection itself (e.g. primaryKeyViolation).
|
||||
- name: "🔧 Resolve the Stalwart management URL and admin login once per run"
|
||||
ansible.builtin.set_fact:
|
||||
stalwart_jmap_client:
|
||||
url: "{{ STALWART_JMAP_URL }}"
|
||||
user_password: "{{ STALWART_ADMIN_USER ~ ':' ~ STALWART_ADMIN_PASSWORD }}"
|
||||
when: stalwart_jmap_client is not defined
|
||||
no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
|
||||
|
||||
- name: "🔧 Stalwart JMAP call"
|
||||
ansible.builtin.shell:
|
||||
cmd: >-
|
||||
container exec -i {{ STALWART_CONTAINER_ADDRESS }}
|
||||
{{ 30 | curl }} -sS -f -K -
|
||||
-H "Content-Type:application/json"
|
||||
{{ STALWART_JMAP_URL }}
|
||||
{{ stalwart_jmap_client.url }}
|
||||
stdin: |-
|
||||
user = {{ (STALWART_ADMIN_USER ~ ':' ~ STALWART_ADMIN_PASSWORD) | to_json(ensure_ascii=False) }}
|
||||
user = {{ stalwart_jmap_client.user_password | to_json(ensure_ascii=False) }}
|
||||
data-binary = {{ stalwart_jmap_body | to_json(ensure_ascii=False) }}
|
||||
register: stalwart_jmap_raw
|
||||
changed_when: false
|
||||
|
||||
@@ -15,7 +15,6 @@ define('YOURLS_UPGRADING', true);
|
||||
|
||||
$infinito_upgrade_done = false;
|
||||
|
||||
// Exception: YOURLS reports bootstrap and upgrade errors with die(), which exits 0; only this handler turns them into a failure.
|
||||
register_shutdown_function(static function (): void {
|
||||
if ($GLOBALS['infinito_upgrade_done']) {
|
||||
return;
|
||||
|
||||
@@ -53,7 +53,7 @@ git worktree add "${path}" "${branch}"
|
||||
|
||||
git_common_dir="$(git rev-parse --path-format=absolute --git-common-dir)"
|
||||
|
||||
lab_octet="$((244 + slot))"
|
||||
lab_prefix="10.160.${slot}"
|
||||
|
||||
echo ">>> Pinning slot ${slot} in ${path}/custom.env"
|
||||
cat >"${path}/custom.env" <<PINS
|
||||
@@ -64,12 +64,12 @@ INFINITO_GATEWAY=172.30.${slot}.1
|
||||
INFINITO_DNS_IP=172.30.${slot}.53
|
||||
INFINITO_IP4=172.30.${slot}.10
|
||||
INFINITO_BIND_IP=127.0.0.$((slot + 1))
|
||||
INFINITO_SWARM_LAB_SUBNET=192.168.${lab_octet}.0/24
|
||||
INFINITO_SWARM_MGR_IP=192.168.${lab_octet}.10
|
||||
INFINITO_SWARM_WRK1_IP=192.168.${lab_octet}.11
|
||||
INFINITO_SWARM_WRK2_IP=192.168.${lab_octet}.12
|
||||
INFINITO_SWARM_NFS_IP=192.168.${lab_octet}.13
|
||||
INFINITO_SWARM_BACKUP_IP=192.168.${lab_octet}.14
|
||||
INFINITO_SWARM_LAB_SUBNET=${lab_prefix}.0/24
|
||||
INFINITO_SWARM_MGR_IP=${lab_prefix}.10
|
||||
INFINITO_SWARM_WRK1_IP=${lab_prefix}.11
|
||||
INFINITO_SWARM_WRK2_IP=${lab_prefix}.12
|
||||
INFINITO_SWARM_NFS_IP=${lab_prefix}.13
|
||||
INFINITO_SWARM_BACKUP_IP=${lab_prefix}.14
|
||||
INFINITO_RUNNER_PREFIX=infinito-${slug}
|
||||
INFINITO_CONTAINER=infinito_nexus_${slug//[.-]/_}
|
||||
INFINITO_CACHE_NETWORK=${cache_network}
|
||||
@@ -86,12 +86,12 @@ env -i -C "${path}" \
|
||||
INFINITO_DNS_IP="172.30.${slot}.53" \
|
||||
INFINITO_IP4="172.30.${slot}.10" \
|
||||
INFINITO_BIND_IP="127.0.0.$((slot + 1))" \
|
||||
INFINITO_SWARM_LAB_SUBNET="192.168.${lab_octet}.0/24" \
|
||||
INFINITO_SWARM_MGR_IP="192.168.${lab_octet}.10" \
|
||||
INFINITO_SWARM_WRK1_IP="192.168.${lab_octet}.11" \
|
||||
INFINITO_SWARM_WRK2_IP="192.168.${lab_octet}.12" \
|
||||
INFINITO_SWARM_NFS_IP="192.168.${lab_octet}.13" \
|
||||
INFINITO_SWARM_BACKUP_IP="192.168.${lab_octet}.14" \
|
||||
INFINITO_SWARM_LAB_SUBNET="${lab_prefix}.0/24" \
|
||||
INFINITO_SWARM_MGR_IP="${lab_prefix}.10" \
|
||||
INFINITO_SWARM_WRK1_IP="${lab_prefix}.11" \
|
||||
INFINITO_SWARM_WRK2_IP="${lab_prefix}.12" \
|
||||
INFINITO_SWARM_NFS_IP="${lab_prefix}.13" \
|
||||
INFINITO_SWARM_BACKUP_IP="${lab_prefix}.14" \
|
||||
INFINITO_RUNNER_PREFIX="infinito-${slug}" \
|
||||
INFINITO_CACHE_NETWORK="${cache_network}" \
|
||||
python3 -m cli.meta.env
|
||||
|
||||
@@ -101,8 +101,6 @@ fi
|
||||
cmd+=(--cache-server-addr 127.0.0.1)
|
||||
cmd+=(--artifact-server-addr 127.0.0.1)
|
||||
if [[ "${ACT_BIND}" != "true" ]] && compgen -G "${_act_repo_root}/roles/web-app-cst-*" >/dev/null; then
|
||||
# Exception: act copies the checkout through .gitignore, which drops the
|
||||
# pulled custom roles, so the job binds the checkout to keep them.
|
||||
echo "act: binding the checkout to carry the pulled custom roles (roles/web-app-cst-*)"
|
||||
ACT_BIND=true
|
||||
fi
|
||||
|
||||
@@ -97,19 +97,24 @@ if [ -n "${CACHE_NET}" ]; then
|
||||
cache_prefix="${cache_subnet%.0/24}"
|
||||
|
||||
node_octet=$((200 + INFINITO_INSTANCE * 5))
|
||||
pin_nodes=true
|
||||
if [ "$((node_octet + 4))" -gt 254 ]; then
|
||||
echo "FAILURE: INFINITO_INSTANCE=${INFINITO_INSTANCE} pushes the node band past ${cache_prefix}.254" >&2
|
||||
exit 1
|
||||
echo "==> INFINITO_INSTANCE=${INFINITO_INSTANCE} leaves no pinned node band in ${cache_prefix}.0/24; Docker assigns the node addresses from ${CACHE_NET}'s pool" >&2
|
||||
pin_nodes=false
|
||||
fi
|
||||
|
||||
for node in "${MGR}" "${WRK1}" "${WRK2}" "${NFS_SERVER}" "${BACKUP_NODE}"; do
|
||||
if ! connect_err="$(docker network connect --ip "${cache_prefix}.${node_octet}" "${CACHE_NET}" "${node}" 2>&1)"; then
|
||||
connect_args=()
|
||||
if [ "${pin_nodes}" = true ]; then
|
||||
connect_args=(--ip "${cache_prefix}.${node_octet}")
|
||||
fi
|
||||
if ! connect_err="$(docker network connect "${connect_args[@]}" "${CACHE_NET}" "${node}" 2>&1)"; then
|
||||
case "${connect_err}" in
|
||||
*"already exists in network"* | *"already attached to network"*)
|
||||
echo "==> ${node} already on ${CACHE_NET}; keeping its current address" >&2
|
||||
;;
|
||||
*)
|
||||
echo "FAILURE: cannot pin ${node} to ${cache_prefix}.${node_octet} on ${CACHE_NET}: ${connect_err}" >&2
|
||||
echo "FAILURE: cannot attach ${node} to ${CACHE_NET} (${connect_args[*]:-pool address}): ${connect_err}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -191,17 +191,17 @@
|
||||
}}
|
||||
when: _package_batch_ids | length > 0
|
||||
|
||||
- name: "🥾 Bootstrap svc-swarm-node"
|
||||
ansible.builtin.include_role:
|
||||
name: svc-swarm-node
|
||||
when: "'svc-swarm-node' | application_allowed(group_names, lookup('deployment').whitelist)"
|
||||
|
||||
- name: "📂 Open service-loader preload context BEFORE Bootstrap"
|
||||
ansible.builtin.set_fact:
|
||||
service_loader_post_run: []
|
||||
service_loader_preloading: true
|
||||
changed_when: false
|
||||
|
||||
- name: "🥾 Bootstrap svc-swarm-node"
|
||||
ansible.builtin.include_role:
|
||||
name: svc-swarm-node
|
||||
when: "'svc-swarm-node' | application_allowed(group_names, lookup('deployment').whitelist)"
|
||||
|
||||
- name: "🥾 Bootstrap NFS clients + registry + sandbox runtimes (MUST precede Preload)"
|
||||
ansible.builtin.include_tasks: ./tasks/utils/setup/group.yml
|
||||
loop:
|
||||
@@ -224,6 +224,10 @@
|
||||
ansible.builtin.include_role:
|
||||
name: sys-svc-container
|
||||
tasks_from: utils/prepull/start.yml
|
||||
vars:
|
||||
sys_svc_container_prepull_extra: >-
|
||||
{{ [lookup('config', 'test-e2e-playwright', 'services.playwright.image') ~ ':' ~ lookup('config', 'test-e2e-playwright', 'services.playwright.version')]
|
||||
if RUNTIME in ['dev', 'act', 'github'] and 'server' in (group_names | get_deployment_types_from_groups) else [] }}
|
||||
|
||||
- name: 🛠️ Preload shared services
|
||||
ansible.builtin.include_role:
|
||||
|
||||
@@ -18,3 +18,4 @@
|
||||
|
||||
- name: "🧊 Carry the applications SPOT past nested {{ load_app_id }}"
|
||||
applications_carrier: {}
|
||||
run_once: false
|
||||
@@ -1,16 +1,29 @@
|
||||
import glob
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from ansible.parsing.dataloader import DataLoader
|
||||
from ansible.template import Templar, trust_as_template
|
||||
|
||||
from utils.cache.applications import get_application_defaults
|
||||
from utils.cache.files import read_text
|
||||
from utils.cache.yaml import load_yaml_any
|
||||
from utils.roles.applications.config import get as get_app_conf
|
||||
from utils.roles.applications.services.registry import is_explicit_truth
|
||||
from utils.roles.entity.name import get_entity_name
|
||||
from utils.roles.mapping import (
|
||||
ROLE_FILE_DEFAULTS_MAIN,
|
||||
ROLE_FILE_META_SERVICES,
|
||||
ROLE_FILE_META_VOLUMES,
|
||||
ROLE_FILE_TEMPL_COMPOSE,
|
||||
ROLE_FILE_VARS_MAIN,
|
||||
)
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
@@ -18,11 +31,452 @@ from . import PROJECT_ROOT
|
||||
PROMETHEUS_APP_ID = "web-app-prometheus"
|
||||
ROLES_DIR = PROJECT_ROOT / "roles"
|
||||
|
||||
_DOMAIN = "infinito.test"
|
||||
_MODES = ("compose", "swarm")
|
||||
_GROUP_NAMES = (
|
||||
"web-app-prometheus",
|
||||
"web-app-gitea",
|
||||
"web-app-matrix",
|
||||
"svc-prx-openresty",
|
||||
)
|
||||
_INJ_OFF = {
|
||||
"logout": False,
|
||||
"dashboard": False,
|
||||
"css": False,
|
||||
"matomo": False,
|
||||
"javascript": False,
|
||||
}
|
||||
_CONTAINER_NAME_SET = re.compile(
|
||||
r'^\s*set \$container_name "([^"]*)";\s*$', re.MULTILINE
|
||||
)
|
||||
_LUA_BLOCK_OPEN = re.compile(r"\b([a-z_]+_by_lua_block)\s*\{")
|
||||
_DOCKER_SOCKET = "unix:/var/run/docker.sock"
|
||||
_DOCKER_PS = {
|
||||
"compose": [
|
||||
{
|
||||
"Names": ["/gitea"],
|
||||
"State": "running",
|
||||
"Status": "Up 3 minutes (unhealthy)",
|
||||
"Labels": {"com.docker.compose.service": "gitea"},
|
||||
},
|
||||
{
|
||||
"Names": ["/prometheus"],
|
||||
"State": "running",
|
||||
"Status": "Up 2 hours (healthy)",
|
||||
"Labels": {"com.docker.compose.service": "prometheus"},
|
||||
},
|
||||
{
|
||||
"Names": ["/matrix-synapse"],
|
||||
"State": "running",
|
||||
"Status": "Up 5 hours",
|
||||
"Labels": None,
|
||||
},
|
||||
],
|
||||
"swarm": [
|
||||
{
|
||||
"Names": ["/gitea_gitea.1.k3j2x"],
|
||||
"State": "running",
|
||||
"Status": "Up 3 minutes (unhealthy)",
|
||||
"Labels": {
|
||||
"com.docker.stack.namespace": "gitea",
|
||||
"com.docker.swarm.service.name": "gitea_gitea",
|
||||
},
|
||||
},
|
||||
{
|
||||
"Names": ["/prometheus_prometheus.1.x9q2"],
|
||||
"State": "running",
|
||||
"Status": "Up 2 hours (healthy)",
|
||||
"Labels": {"com.docker.swarm.service.name": "prometheus_prometheus"},
|
||||
},
|
||||
{
|
||||
"Names": ["/matrix_synapse.1.old"],
|
||||
"State": "running",
|
||||
"Status": "Up 1 hour (unhealthy)",
|
||||
"Labels": {"com.docker.swarm.service.name": "matrix_synapse"},
|
||||
},
|
||||
{
|
||||
"Names": ["/matrix_synapse.2.new"],
|
||||
"State": "running",
|
||||
"Status": "Up 1 hour (healthy)",
|
||||
"Labels": {"com.docker.swarm.service.name": "matrix_synapse"},
|
||||
},
|
||||
],
|
||||
}
|
||||
_LUA_HARNESS = r"""
|
||||
local dir = arg[1]
|
||||
local null = newproxy(false)
|
||||
|
||||
local function read(name)
|
||||
local f = assert(io.open(dir .. "/" .. name))
|
||||
local s = f:read("*a")
|
||||
f:close()
|
||||
return s
|
||||
end
|
||||
|
||||
local body = read("containers.json")
|
||||
local decoded = assert(loadfile(dir .. "/containers.lua"))(null)
|
||||
|
||||
local function world(ptype, connect_err)
|
||||
local w = {connects = 0, warns = 0, now = 0, at = {}, every = {}, store = {}, ttl = {}, expires = {}, gauges = {}}
|
||||
local function live(k) return w.store[k] ~= nil and (w.expires[k] == 0 or w.expires[k] > w.now) end
|
||||
local function metric(name)
|
||||
return {
|
||||
set = function(_, v) w.gauges[name] = v end,
|
||||
inc = function() end,
|
||||
observe = function() end,
|
||||
}
|
||||
end
|
||||
local prom = {
|
||||
counter = function(_, n) return metric(n) end,
|
||||
histogram = function(_, n) return metric(n) end,
|
||||
gauge = function(_, n) return metric(n) end,
|
||||
collect = function() w.collected = true end,
|
||||
}
|
||||
local modules = {
|
||||
["ngx.process"] = {
|
||||
type = function() return ptype end,
|
||||
enable_privileged_agent = function() w.enabled = true return true end,
|
||||
},
|
||||
["resty.prometheus"] = {init = function() return prom end},
|
||||
["cjson.safe"] = {decode = function(s) if s == body then return decoded end end},
|
||||
}
|
||||
local ngx = {
|
||||
ERR = 4, WARN = 5, HTTP_OK = 200, HTTP_SERVICE_UNAVAILABLE = 503,
|
||||
log = function(level) if level == 5 then w.warns = w.warns + 1 end end,
|
||||
header = {},
|
||||
var = {},
|
||||
shared = {health_containers = {
|
||||
set = function(_, k, v, ttl)
|
||||
w.store[k] = v w.ttl[k] = ttl w.expires[k] = ttl == 0 and 0 or w.now + ttl
|
||||
return true
|
||||
end,
|
||||
get = function(_, k) if live(k) then return w.store[k] end end,
|
||||
get_keys = function()
|
||||
local keys = {}
|
||||
for k in pairs(w.store) do if live(k) then keys[#keys + 1] = k end end
|
||||
return keys
|
||||
end,
|
||||
}},
|
||||
timer = {
|
||||
at = function(delay, fn) w.at[#w.at + 1] = delay fn(false) return true end,
|
||||
every = function(delay) w.every[#w.every + 1] = delay return true end,
|
||||
},
|
||||
socket = {tcp = function()
|
||||
local lines = {"HTTP/1.0 200 OK", "Api-Version: 1.41", "Content-Type: application/json", ""}
|
||||
local n = 0
|
||||
return {
|
||||
connect = function(_, target)
|
||||
w.connects = w.connects + 1 w.target = target
|
||||
if connect_err then return nil, connect_err end
|
||||
return true
|
||||
end,
|
||||
send = function(_, data) w.request = data return #data end,
|
||||
receive = function(_, pattern)
|
||||
if pattern == "*l" then n = n + 1 return lines[n] end
|
||||
return body
|
||||
end,
|
||||
close = function() end,
|
||||
}
|
||||
end},
|
||||
location = {capture = function() return {status = w.probe} end},
|
||||
worker = {id = function() if ptype == "worker" then return 0 end end},
|
||||
}
|
||||
ngx.print = function() end
|
||||
w.ngx = ngx
|
||||
w.env = setmetatable({ngx = ngx, require = function(name) return assert(modules[name], name) end}, {__index = _G})
|
||||
return w
|
||||
end
|
||||
|
||||
local function run(w, name)
|
||||
local chunk = assert(loadstring(read(name), name))
|
||||
setfenv(chunk, w.env)
|
||||
chunk()
|
||||
end
|
||||
|
||||
local master = world("master")
|
||||
run(master, "init.lua")
|
||||
print("enabled\t" .. tostring(master.enabled))
|
||||
|
||||
local worker = world("worker")
|
||||
run(worker, "init_worker.lua")
|
||||
print("worker\t" .. worker.connects .. "/" .. #worker.at .. "/" .. #worker.every)
|
||||
|
||||
local agent = world("privileged agent")
|
||||
run(agent, "init_worker.lua")
|
||||
print("agent\t" .. tostring(agent.target) .. "\t" .. tostring(agent.request and agent.request:match("^[^\r\n]*")) .. "\t" .. table.concat(agent.at, ",") .. "/" .. table.concat(agent.every, ","))
|
||||
for key, value in pairs(agent.store) do
|
||||
print("dict\t" .. key .. "\t" .. value .. "/" .. agent.ttl[key])
|
||||
end
|
||||
|
||||
local function request(w, key)
|
||||
w.ngx.var = {container_name = key, host = "h", app_id = "a"}
|
||||
w.probe = 200
|
||||
w.ngx.status = nil
|
||||
run(w, "live.lua")
|
||||
run(w, "metricz.lua")
|
||||
return tostring(w.ngx.status), tostring(w.gauges.nginx_stack_up), tostring(w.gauges.nginx_docker_health_entries)
|
||||
end
|
||||
|
||||
for i = 2, #arg do
|
||||
local status, up, entries = request(agent, arg[i])
|
||||
print("live\t" .. arg[i] .. "\t" .. status)
|
||||
print("metricz\t" .. arg[i] .. "\t" .. up .. "\t" .. entries)
|
||||
end
|
||||
|
||||
agent.now = 61
|
||||
local _, _, expired = request(agent, arg[2])
|
||||
print("expired\t" .. expired)
|
||||
|
||||
local denied = world("privileged agent", "permission denied")
|
||||
run(denied, "init_worker.lua")
|
||||
print("denied\t" .. denied.connects .. "\t" .. denied.warns)
|
||||
for i = 2, #arg do
|
||||
local status, up, entries = request(denied, arg[i])
|
||||
print("denied_metricz\t" .. arg[i] .. "\t" .. status .. "\t" .. up .. "\t" .. entries)
|
||||
end
|
||||
|
||||
local stale = world("worker")
|
||||
run(stale, "init_worker.lua")
|
||||
stale.env.metric_docker_health_entries = nil
|
||||
local stale_ok = pcall(request, stale, arg[2])
|
||||
print("stale_conf\t" .. tostring(stale_ok) .. "\t" .. tostring(stale.collected))
|
||||
"""
|
||||
|
||||
|
||||
def _load_config(file_path: str) -> dict:
|
||||
return load_yaml_any(file_path) or {}
|
||||
|
||||
|
||||
def _trusted(value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
return trust_as_template(value)
|
||||
if isinstance(value, Mapping):
|
||||
return {k: _trusted(v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_trusted(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def _vars_layer(paths: list[Path]) -> dict:
|
||||
loaded: dict = {}
|
||||
for path in paths:
|
||||
if not path.is_file():
|
||||
continue
|
||||
data = load_yaml_any(str(path), default_if_missing={})
|
||||
if isinstance(data, Mapping):
|
||||
loaded.update({k: _trusted(v) for k, v in data.items()})
|
||||
return loaded
|
||||
|
||||
|
||||
def _render(application_id: str, template: str, mode: str, extra: dict) -> str:
|
||||
"""Render a template through Ansible's templar with the deploy's own lookups.
|
||||
|
||||
Args:
|
||||
application_id: role whose defaults and vars layer over group_vars/all.
|
||||
template: template path relative to the repository root.
|
||||
mode: DEPLOYMENT_MODE, ``compose`` or ``swarm``.
|
||||
extra: variables the including task passes to the template.
|
||||
|
||||
Returns:
|
||||
The rendered text.
|
||||
"""
|
||||
source = (PROJECT_ROOT / template).resolve()
|
||||
return _template(
|
||||
application_id, mode, extra, f"{{{{ lookup('template', '{source}') }}}}"
|
||||
)
|
||||
|
||||
|
||||
def _template(application_id: str, mode: str, extra: dict, expression: str) -> str:
|
||||
"""Evaluate a Jinja expression with the variables a role deploy carries.
|
||||
|
||||
Args:
|
||||
application_id: role whose defaults and vars layer over group_vars/all.
|
||||
mode: DEPLOYMENT_MODE, ``compose`` or ``swarm``.
|
||||
extra: variables layered on top.
|
||||
expression: the Jinja source to evaluate.
|
||||
|
||||
Returns:
|
||||
The evaluated text.
|
||||
"""
|
||||
applications = get_application_defaults(roles_dir=ROLES_DIR)
|
||||
variables = {
|
||||
**_vars_layer(sorted((PROJECT_ROOT / "group_vars" / "all").glob("*.yml"))),
|
||||
**_vars_layer(
|
||||
[
|
||||
ROLES_DIR / application_id / ROLE_FILE_DEFAULTS_MAIN,
|
||||
ROLES_DIR / application_id / ROLE_FILE_VARS_MAIN,
|
||||
]
|
||||
),
|
||||
"applications": applications,
|
||||
"application_id": application_id,
|
||||
"group_names": list(_GROUP_NAMES),
|
||||
"inventory_hostname": "localhost",
|
||||
"playbook_dir": str(PROJECT_ROOT),
|
||||
"ansible_search_path": [str(PROJECT_ROOT)],
|
||||
"DEPLOYMENT_MODE": mode,
|
||||
"TIMEOUT_FACTOR": 1,
|
||||
"PRIMARY_DOMAIN": _DOMAIN,
|
||||
"DOMAIN_PRIMARY": _DOMAIN,
|
||||
"SYSTEM_EMAIL_DOMAIN": _DOMAIN,
|
||||
"inj_enabled": _INJ_OFF,
|
||||
**extra,
|
||||
}
|
||||
templar = Templar(loader=DataLoader(), variables=variables)
|
||||
return str(templar.template(trust_as_template(expression)))
|
||||
|
||||
|
||||
def _vhost(application_id: str, service_key: str, mode: str, **extra) -> str:
|
||||
"""Render the vhost include that sets ``$container_name`` for one application.
|
||||
|
||||
Args:
|
||||
application_id: the proxied application.
|
||||
service_key: the ``services.<key>`` entry whose local http port the vhost proxies.
|
||||
mode: DEPLOYMENT_MODE, ``compose`` or ``swarm``.
|
||||
extra: further template variables, e.g. ``matrix_flavor``.
|
||||
|
||||
Returns:
|
||||
The rendered nginx location block.
|
||||
"""
|
||||
template = (
|
||||
"roles/web-app-matrix/templates/synapse.conf.j2"
|
||||
if application_id == "web-app-matrix"
|
||||
else "roles/web-app-prometheus/templates/nginx/locations.conf.j2"
|
||||
)
|
||||
http_port = get_app_conf(
|
||||
applications=get_application_defaults(roles_dir=ROLES_DIR),
|
||||
application_id=application_id,
|
||||
config_path=f"services.{service_key}.ports.local.http",
|
||||
strict=False,
|
||||
default="",
|
||||
)
|
||||
domain = _template(
|
||||
application_id, mode, {}, "{{ lookup('domain', application_id) }}"
|
||||
)
|
||||
return _render(
|
||||
application_id,
|
||||
template,
|
||||
mode,
|
||||
{"http_port": http_port, "domain": domain, **extra},
|
||||
)
|
||||
|
||||
|
||||
def _container_name(rendered: str) -> str:
|
||||
found = _CONTAINER_NAME_SET.findall(rendered)
|
||||
if len(found) != 1:
|
||||
raise AssertionError(f"expected one 'set $container_name', found {found!r}")
|
||||
return found[0]
|
||||
|
||||
|
||||
def _lua_blocks(rendered: str) -> dict[str, list[str]]:
|
||||
"""Collect the Lua body of every ``*_by_lua_block`` in rendered nginx config.
|
||||
|
||||
Args:
|
||||
rendered: nginx configuration text.
|
||||
|
||||
Returns:
|
||||
Directive name mapped to its bodies in document order.
|
||||
"""
|
||||
blocks: dict[str, list[str]] = {}
|
||||
consumed = 0
|
||||
for match in _LUA_BLOCK_OPEN.finditer(rendered):
|
||||
if match.start() < consumed:
|
||||
continue
|
||||
depth, i, quote = 1, match.end(), None
|
||||
while depth:
|
||||
ch = rendered[i]
|
||||
if quote:
|
||||
if ch == "\\":
|
||||
i += 2
|
||||
continue
|
||||
if ch == quote:
|
||||
quote = None
|
||||
elif rendered.startswith("--", i):
|
||||
i = rendered.index("\n", i)
|
||||
continue
|
||||
elif ch in "\"'":
|
||||
quote = ch
|
||||
elif ch == "{":
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
i += 1
|
||||
consumed = i
|
||||
blocks.setdefault(match.group(1), []).append(rendered[match.end() : i - 1])
|
||||
return blocks
|
||||
|
||||
|
||||
def _lua_literal(value: object) -> str:
|
||||
if value is None:
|
||||
return "null"
|
||||
if isinstance(value, str):
|
||||
return json.dumps(value)
|
||||
if isinstance(value, list):
|
||||
return "{" + ", ".join(_lua_literal(v) for v in value) + "}"
|
||||
if isinstance(value, Mapping):
|
||||
return (
|
||||
"{"
|
||||
+ ", ".join(
|
||||
f"[{json.dumps(k)}] = {_lua_literal(v)}" for k, v in value.items()
|
||||
)
|
||||
+ "}"
|
||||
)
|
||||
raise TypeError(f"no Lua literal for {type(value).__name__}")
|
||||
|
||||
|
||||
def _harness_outcome(mode: str, keys: list[str]) -> dict:
|
||||
"""Run the rendered Lua blocks under luajit against a canned Docker answer.
|
||||
|
||||
Args:
|
||||
mode: DEPLOYMENT_MODE whose renders and ``_DOCKER_PS`` fixture are used.
|
||||
keys: ``$container_name`` values to request ``/healthz/live`` and ``/metricz`` with.
|
||||
|
||||
Returns:
|
||||
``enabled``, ``worker``, ``agent``, ``expired``, ``denied`` and ``stale_conf`` results
|
||||
plus the ``dict``, ``live``, ``metricz`` and ``denied_metricz`` maps
|
||||
keyed by container key, each value a tuple of the printed fields.
|
||||
"""
|
||||
conf = _lua_blocks(
|
||||
_render(
|
||||
PROMETHEUS_APP_ID,
|
||||
"roles/web-app-prometheus/templates/nginx/prometheus.conf.j2",
|
||||
mode,
|
||||
{},
|
||||
)
|
||||
)
|
||||
vhost = _lua_blocks(_vhost(PROMETHEUS_APP_ID, "prometheus", mode))
|
||||
files = {
|
||||
"init.lua": conf["init_by_lua_block"][0],
|
||||
"init_worker.lua": conf["init_worker_by_lua_block"][0],
|
||||
"live.lua": vhost["content_by_lua_block"][0],
|
||||
"metricz.lua": vhost["content_by_lua_block"][-1],
|
||||
"containers.json": json.dumps(_DOCKER_PS[mode]),
|
||||
"containers.lua": "local null = ...\nreturn "
|
||||
+ _lua_literal(_DOCKER_PS[mode])
|
||||
+ "\n",
|
||||
"harness.lua": _LUA_HARNESS,
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
work = Path(tmp)
|
||||
for name, text in files.items():
|
||||
(work / name).write_text(text)
|
||||
result = subprocess.run(
|
||||
["luajit", str(work / "harness.lua"), str(work), *keys],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise AssertionError(f"luajit failed in {mode}:\n{result.stderr}")
|
||||
keyed = ("dict", "live", "metricz", "denied_metricz")
|
||||
outcome: dict = {kind: {} for kind in keyed}
|
||||
for fields in (line.split("\t") for line in result.stdout.splitlines()):
|
||||
if fields[0] in keyed:
|
||||
outcome[fields[0]][fields[1]] = tuple(fields[2:])
|
||||
else:
|
||||
outcome[fields[0]] = fields[1:]
|
||||
return outcome
|
||||
|
||||
|
||||
class TestPrometheusServicePresence(unittest.TestCase):
|
||||
"""
|
||||
All web-app-* and web-svc-* roles (except web-app-prometheus itself) must
|
||||
@@ -642,6 +1096,145 @@ class TestDockerHealthCheck(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestDockerHealthKey(unittest.TestCase):
|
||||
CASES = (
|
||||
("web-app-gitea", "gitea", {}, "gitea"),
|
||||
(PROMETHEUS_APP_ID, "prometheus", {}, "prometheus"),
|
||||
("web-app-matrix", "synapse", {"matrix_flavor": "compose"}, "synapse"),
|
||||
("web-app-matrix", "synapse", {"matrix_flavor": "ansible"}, "matrix"),
|
||||
)
|
||||
|
||||
def test_key_is_the_name_docker_reports_in_each_mode(self):
|
||||
applications = get_application_defaults(roles_dir=ROLES_DIR)
|
||||
for application_id, port_key, extra, docker_key in self.CASES:
|
||||
expected = {
|
||||
"compose": get_app_conf(
|
||||
applications=applications,
|
||||
application_id=application_id,
|
||||
config_path=f"services.{docker_key}.name",
|
||||
),
|
||||
"swarm": f"{get_entity_name(application_id)}_{docker_key}",
|
||||
}
|
||||
for mode in _MODES:
|
||||
with self.subTest(application_id=application_id, mode=mode, **extra):
|
||||
self.assertEqual(
|
||||
expected[mode],
|
||||
_container_name(
|
||||
_vhost(application_id, port_key, mode, **extra)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which("luajit"), "luajit runs the rendered Lua blocks")
|
||||
class TestDockerHealthLuaRuntime(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.keys = {
|
||||
mode: {
|
||||
"unhealthy": _container_name(_vhost("web-app-gitea", "gitea", mode)),
|
||||
"healthy": _container_name(
|
||||
_vhost(PROMETHEUS_APP_ID, "prometheus", mode)
|
||||
),
|
||||
"mixed": _container_name(
|
||||
_vhost("web-app-matrix", "synapse", mode, matrix_flavor="compose")
|
||||
),
|
||||
"unknown": "not-on-this-node",
|
||||
}
|
||||
for mode in _MODES
|
||||
}
|
||||
cls.outcomes = {
|
||||
mode: _harness_outcome(mode, list(cls.keys[mode].values()))
|
||||
for mode in _MODES
|
||||
}
|
||||
|
||||
def test_init_enables_the_privileged_agent(self):
|
||||
for mode in _MODES:
|
||||
with self.subTest(mode=mode):
|
||||
self.assertEqual(["true"], self.outcomes[mode]["enabled"])
|
||||
|
||||
def test_request_workers_never_touch_the_docker_socket(self):
|
||||
for mode in _MODES:
|
||||
with self.subTest(mode=mode):
|
||||
self.assertEqual(["0/0/0"], self.outcomes[mode]["worker"])
|
||||
|
||||
def test_privileged_agent_polls_the_docker_socket(self):
|
||||
for mode in _MODES:
|
||||
with self.subTest(mode=mode):
|
||||
self.assertEqual(
|
||||
[
|
||||
_DOCKER_SOCKET,
|
||||
"GET /v1.41/containers/json HTTP/1.0",
|
||||
"0/15",
|
||||
],
|
||||
self.outcomes[mode]["agent"],
|
||||
)
|
||||
|
||||
def test_poller_stores_state_under_the_key_the_vhost_reads(self):
|
||||
for mode in _MODES:
|
||||
keys = self.keys[mode]
|
||||
with self.subTest(mode=mode):
|
||||
self.assertEqual(
|
||||
{
|
||||
keys["unhealthy"]: ("down/60",),
|
||||
keys["healthy"]: ("up/60",),
|
||||
keys["mixed"]: ("up/60",),
|
||||
},
|
||||
{
|
||||
key: self.outcomes[mode]["dict"].get(key)
|
||||
for key in (keys["unhealthy"], keys["healthy"], keys["mixed"])
|
||||
},
|
||||
)
|
||||
|
||||
def test_live_and_stack_up_report_an_unhealthy_container_down(self):
|
||||
for mode in _MODES:
|
||||
keys = self.keys[mode]
|
||||
outcome = self.outcomes[mode]
|
||||
with self.subTest(mode=mode):
|
||||
self.assertEqual(
|
||||
{
|
||||
keys["unhealthy"]: ("503", "0"),
|
||||
keys["healthy"]: ("200", "1"),
|
||||
keys["mixed"]: ("200", "1"),
|
||||
keys["unknown"]: ("200", "1"),
|
||||
},
|
||||
{
|
||||
key: (outcome["live"][key][0], outcome["metricz"][key][0])
|
||||
for key in keys.values()
|
||||
},
|
||||
)
|
||||
|
||||
def test_entries_gauge_counts_the_unexpired_dict_entries(self):
|
||||
for mode in _MODES:
|
||||
outcome = self.outcomes[mode]
|
||||
with self.subTest(mode=mode):
|
||||
self.assertGreater(len(outcome["dict"]), 0)
|
||||
self.assertEqual(
|
||||
{str(len(outcome["dict"]))},
|
||||
{fields[1] for fields in outcome["metricz"].values()},
|
||||
)
|
||||
|
||||
def test_entries_gauge_drops_to_zero_once_the_poller_stops(self):
|
||||
for mode in _MODES:
|
||||
with self.subTest(mode=mode):
|
||||
self.assertEqual(["0"], self.outcomes[mode]["expired"])
|
||||
|
||||
def test_entries_gauge_is_zero_when_the_socket_is_denied(self):
|
||||
for mode in _MODES:
|
||||
keys = self.keys[mode]
|
||||
outcome = self.outcomes[mode]
|
||||
with self.subTest(mode=mode):
|
||||
self.assertEqual(["1", "1"], outcome["denied"])
|
||||
self.assertEqual(
|
||||
dict.fromkeys(keys.values(), ("200", "1", "0")),
|
||||
outcome["denied_metricz"],
|
||||
)
|
||||
|
||||
def test_metricz_still_collects_before_the_gauge_is_registered(self):
|
||||
for mode in _MODES:
|
||||
with self.subTest(mode=mode):
|
||||
self.assertEqual(["true", "true"], self.outcomes[mode]["stale_conf"])
|
||||
|
||||
|
||||
class TestNativeAppMetrics(unittest.TestCase):
|
||||
"""
|
||||
Applications that provide native Prometheus metrics MUST have a scrape job
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Every worktree slot must pin a valid swarm lab subnet of its own.
|
||||
|
||||
`scripts/system/worktree/up.sh` derives `INFINITO_SWARM_LAB_SUBNET` from the
|
||||
slot. A derivation that overflows an octet yields an address Docker rejects at
|
||||
network creation, and one that lands in a reserved range breaks the swarm at
|
||||
runtime: inside NETWORK_TOR_VIRTUAL_CIDR svc-net-tor redirects every TCP
|
||||
connection to Tor's TransPort, so the nodes cannot reach each other.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import re
|
||||
import unittest
|
||||
|
||||
from utils.cache.files import read_text
|
||||
from utils.cache.yaml import load_yaml_any
|
||||
from utils.env.parser import parse_static_env
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
_UP_SH = PROJECT_ROOT / "scripts/system/worktree/up.sh"
|
||||
_NETWORKS = PROJECT_ROOT / "group_vars/all/08_networks.yml"
|
||||
_MAX_SLOT_RE = re.compile(r"^max_slot=(?P<value>\d+)$", re.MULTILINE)
|
||||
_LAB_PREFIX_RE = re.compile(r'^lab_prefix="(?P<template>[^"]+)"$', re.MULTILINE)
|
||||
_LAB_SUBNET_RE = re.compile(
|
||||
r"^INFINITO_SWARM_LAB_SUBNET=\$\{lab_prefix\}\.0/24$", re.MULTILINE
|
||||
)
|
||||
|
||||
|
||||
def _reserved_networks() -> dict[str, ipaddress.IPv4Network]:
|
||||
networks = load_yaml_any(str(_NETWORKS), default_if_missing={})
|
||||
reserved = {
|
||||
"NETWORK_TOR_VIRTUAL_CIDR": networks["NETWORK_TOR_VIRTUAL_CIDR"],
|
||||
"NETWORK_MESH_POOL": networks["NETWORK_MESH_POOL"],
|
||||
}
|
||||
for index, pool in enumerate(networks["NETWORK_DOCKER_ADDRESS_POOLS"]):
|
||||
reserved[f"NETWORK_DOCKER_ADDRESS_POOLS[{index}]"] = pool["base"]
|
||||
for index, pool in enumerate(networks["NETWORK_NESTED_DOCKER_ADDRESS_POOLS"]):
|
||||
reserved[f"NETWORK_NESTED_DOCKER_ADDRESS_POOLS[{index}]"] = pool["base"]
|
||||
reserved["NETWORK_NESTED_DOCKER_BIP"] = networks["NETWORK_NESTED_DOCKER_BIP"]
|
||||
reserved["primary INFINITO_SWARM_LAB_SUBNET"] = parse_static_env(
|
||||
PROJECT_ROOT / "default.env"
|
||||
)["INFINITO_SWARM_LAB_SUBNET"]
|
||||
reserved["worktree INFINITO_SUBNET range"] = "172.30.0.0/16"
|
||||
return {
|
||||
name: ipaddress.ip_network(value, strict=False)
|
||||
for name, value in reserved.items()
|
||||
}
|
||||
|
||||
|
||||
class TestWorktreeSwarmLabSubnet(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
source = read_text(str(_UP_SH))
|
||||
max_slot = _MAX_SLOT_RE.search(source)
|
||||
lab_prefix = _LAB_PREFIX_RE.search(source)
|
||||
self.assertIsNotNone(max_slot, f"{_UP_SH} declares no max_slot")
|
||||
self.assertIsNotNone(lab_prefix, f"{_UP_SH} declares no lab_prefix")
|
||||
self.assertRegex(source, _LAB_SUBNET_RE)
|
||||
self.max_slot = int(max_slot.group("value"))
|
||||
self.template = lab_prefix.group("template")
|
||||
self.reserved = _reserved_networks()
|
||||
|
||||
def _subnet(self, slot: int) -> ipaddress.IPv4Network:
|
||||
prefix = self.template.replace("${slot}", str(slot))
|
||||
return ipaddress.ip_network(f"{prefix}.0/24")
|
||||
|
||||
def test_every_slot_pins_a_valid_distinct_unreserved_lab_subnet(self) -> None:
|
||||
seen: dict[ipaddress.IPv4Network, int] = {}
|
||||
for slot in range(1, self.max_slot + 1):
|
||||
with self.subTest(slot=slot):
|
||||
subnet = self._subnet(slot)
|
||||
clashes = sorted(
|
||||
name
|
||||
for name, network in self.reserved.items()
|
||||
if subnet.overlaps(network)
|
||||
)
|
||||
self.assertEqual(
|
||||
clashes, [], f"slot {slot} lab {subnet} overlaps {clashes}"
|
||||
)
|
||||
self.assertNotIn(
|
||||
subnet, seen, f"slot {slot} repeats slot {seen.get(subnet)}"
|
||||
)
|
||||
seen[subnet] = slot
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -10,10 +10,13 @@ from unittest.mock import patch
|
||||
|
||||
from ansible.errors import AnsibleError
|
||||
|
||||
from plugins.lookup.applications import LookupModule as ApplicationsLookup
|
||||
from plugins.lookup.email import LookupModule
|
||||
from utils.cache import _reset_cache_for_tests
|
||||
from utils.cache import applications as cache_applications
|
||||
from utils.cache import base as cache_base
|
||||
from utils.cache import users as cache_users
|
||||
from utils.cache.carrier import APPLICATIONS_RENDERED_FACT
|
||||
from utils.cache.yaml import dump_yaml_str
|
||||
from utils.roles.mapping import ROLE_FILE_META_DOMAINS, ROLE_FILE_META_SERVICES
|
||||
|
||||
@@ -511,6 +514,88 @@ class TestProviderOnionIsClusterWide(unittest.TestCase):
|
||||
|
||||
self.assertTrue(result["tls"])
|
||||
|
||||
def _park(self, variables: dict) -> dict:
|
||||
lookup = ApplicationsLookup()
|
||||
lookup._templar = self.lookup._templar
|
||||
return lookup.run(
|
||||
[], variables=variables, roles_dir=str(self._tmp / "roles"), carrier=True
|
||||
)[0]
|
||||
|
||||
def _provider_view(self, worker: dict, **overrides) -> dict:
|
||||
view = dict(worker, inventory_hostname="mgr")
|
||||
view["group_names"] = worker["hostvars"]["mgr"]["group_names"]
|
||||
view.update(overrides)
|
||||
return view
|
||||
|
||||
def _worker_with_carrier(self) -> dict:
|
||||
worker = self._worker()
|
||||
worker[APPLICATIONS_RENDERED_FACT] = self._park(worker)
|
||||
return worker
|
||||
|
||||
def _run_counting_renders(self, worker: dict) -> tuple[dict, int]:
|
||||
_reset_cache_for_tests()
|
||||
with patch.object(
|
||||
cache_applications,
|
||||
"_render_with_templar",
|
||||
wraps=cache_applications._render_with_templar,
|
||||
) as render:
|
||||
result = self._run(worker)
|
||||
return result, render.call_count
|
||||
|
||||
def test_worker_reads_the_provider_view_from_the_provider_carrier(self) -> None:
|
||||
self._seed(ports=self.STALWART_PORTS)
|
||||
worker = self._worker_with_carrier()
|
||||
expected, baseline_renders = self._run_counting_renders(worker)
|
||||
worker["hostvars"]["mgr"][APPLICATIONS_RENDERED_FACT] = self._park(
|
||||
self._provider_view(worker)
|
||||
)
|
||||
|
||||
result, renders = self._run_counting_renders(worker)
|
||||
|
||||
self.assertEqual(baseline_renders, 1)
|
||||
self.assertEqual(renders, 0)
|
||||
self.assertEqual(result, expected)
|
||||
|
||||
def test_worker_renders_when_the_provider_carrier_key_differs(self) -> None:
|
||||
self._seed(ports=self.STALWART_PORTS)
|
||||
worker = self._worker_with_carrier()
|
||||
expected, _ = self._run_counting_renders(worker)
|
||||
worker["hostvars"]["mgr"][APPLICATIONS_RENDERED_FACT] = self._park(
|
||||
self._provider_view(worker, group_names=["web-app-mailprov"])
|
||||
)
|
||||
|
||||
result, renders = self._run_counting_renders(worker)
|
||||
|
||||
self.assertGreater(renders, 0)
|
||||
self.assertEqual(result, expected)
|
||||
self.assertFalse(result["tls"])
|
||||
|
||||
def test_worker_reads_the_provider_carrier_untemplated(self) -> None:
|
||||
self._seed(ports=self.STALWART_PORTS)
|
||||
worker = self._worker_with_carrier()
|
||||
expected, _ = self._run_counting_renders(worker)
|
||||
raw = {
|
||||
"mgr": dict(
|
||||
worker["hostvars"]["mgr"],
|
||||
**{APPLICATIONS_RENDERED_FACT: self._park(self._provider_view(worker))},
|
||||
)
|
||||
}
|
||||
worker["hostvars"] = _RawGetHostvars(worker["hostvars"], raw)
|
||||
|
||||
result, renders = self._run_counting_renders(worker)
|
||||
|
||||
self.assertEqual(renders, 0)
|
||||
self.assertEqual(result, expected)
|
||||
|
||||
|
||||
class _RawGetHostvars(dict):
|
||||
def __init__(self, templated: dict, raw: dict) -> None:
|
||||
super().__init__(templated)
|
||||
self._raw = raw
|
||||
|
||||
def raw_get(self, host: str) -> dict:
|
||||
return self._raw[host]
|
||||
|
||||
|
||||
class _DummyTemplar:
|
||||
def __init__(self, available_variables: dict[str, str]) -> None:
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unittest
|
||||
|
||||
from utils.cache.files import read_text
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
ROLE = PROJECT_ROOT / "roles" / "svc-net-tor"
|
||||
TEST_ENV = ROLE / "templates" / "test.env.j2"
|
||||
TORRC = ROLE / "templates" / "torrc.j2"
|
||||
TOR_DEFAULT_SOCKS_TIMEOUT = 120
|
||||
|
||||
|
||||
class TestOnionPortWindow(unittest.TestCase):
|
||||
def test_an_attempt_outlasts_tors_socks_timeout(self) -> None:
|
||||
self.assertNotRegex(
|
||||
read_text(str(TORRC)),
|
||||
r"(?m)^SocksTimeout\b",
|
||||
"torrc now sets SocksTimeout; ONION_PORT_TIMEOUT has to be derived "
|
||||
"from it instead of from Tor's default",
|
||||
)
|
||||
declared = re.search(
|
||||
r"(?m)^ONION_PORT_TIMEOUT=(\d+)$", read_text(str(TEST_ENV))
|
||||
)
|
||||
self.assertIsNotNone(declared, "test.env.j2 declares no ONION_PORT_TIMEOUT")
|
||||
self.assertGreater(
|
||||
int(declared.group(1)),
|
||||
TOR_DEFAULT_SOCKS_TIMEOUT,
|
||||
"a port probe attempt that ends before Tor's SocksTimeout walks away "
|
||||
"from a rendezvous Tor is still building, and Tor never reaches its "
|
||||
"own verdict or logs it",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -9,6 +9,7 @@ secrets, so neither may reach curl's command line.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import unittest
|
||||
from typing import ClassVar
|
||||
|
||||
@@ -24,6 +25,14 @@ TASK = PROJECT_ROOT / "roles" / "web-app-stalwart" / "tasks" / "jmap_call.yml"
|
||||
|
||||
PASSWORD = 'p"a\\s$s`w#o[r]d é'
|
||||
|
||||
SWARM_ADDRESS = '"$(/usr/bin/resolve-container-id stalwart stalwart)"'
|
||||
|
||||
RESOLVED_ONCE = (
|
||||
"STALWART_JMAP_URL",
|
||||
"STALWART_ADMIN_USER",
|
||||
"STALWART_ADMIN_PASSWORD",
|
||||
)
|
||||
|
||||
|
||||
def _tasks() -> list[dict]:
|
||||
return load_yaml_any(str(TASK), default_if_missing=[])
|
||||
@@ -37,6 +46,15 @@ def _call_task() -> dict:
|
||||
return next(_module(t, "shell") for t in _tasks() if _module(t, "shell"))
|
||||
|
||||
|
||||
def _client_task() -> dict:
|
||||
return next(t for t in _tasks() if "stalwart_jmap_client" in _module(t, "set_fact"))
|
||||
|
||||
|
||||
def _resolved_client(**variables) -> dict:
|
||||
client = _module(_client_task(), "set_fact")["stalwart_jmap_client"]
|
||||
return {key: _render(raw, **variables) for key, raw in client.items()}
|
||||
|
||||
|
||||
def _assert_task() -> dict:
|
||||
return next(t for t in _tasks() if _module(t, "assert"))
|
||||
|
||||
@@ -220,16 +238,68 @@ class TestJmapSecretsStayOffArgv(unittest.TestCase):
|
||||
|
||||
def test_stdin_config_round_trips_hostile_values(self):
|
||||
body = json.dumps({"secret": 'q"\\x', "name": "Zoë"})
|
||||
rendered = _render(
|
||||
self.task["stdin"],
|
||||
client = _resolved_client(
|
||||
STALWART_JMAP_URL="http://localhost:8080/jmap/",
|
||||
STALWART_ADMIN_USER="stalwartadmin",
|
||||
STALWART_ADMIN_PASSWORD=PASSWORD,
|
||||
stalwart_jmap_body=body,
|
||||
)
|
||||
rendered = _render(
|
||||
self.task["stdin"], stalwart_jmap_client=client, stalwart_jmap_body=body
|
||||
)
|
||||
config = dict(line.split(" = ", 1) for line in rendered.splitlines())
|
||||
self.assertEqual(json.loads(config["user"]), f"stalwartadmin:{PASSWORD}")
|
||||
self.assertEqual(json.loads(config["data-binary"]), body)
|
||||
|
||||
|
||||
class TestJmapClientResolvedOncePerRun(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.call = _call_task()
|
||||
self.resolve = _client_task()
|
||||
|
||||
def test_the_call_templates_no_resolved_once_variable(self):
|
||||
for field in ("cmd", "stdin"):
|
||||
for name in RESOLVED_ONCE:
|
||||
self.assertNotIn(name, self.call[field])
|
||||
|
||||
def test_the_client_carries_every_resolved_once_variable(self):
|
||||
client = _module(self.resolve, "set_fact")["stalwart_jmap_client"]
|
||||
for name in RESOLVED_ONCE:
|
||||
self.assertTrue(any(name in raw for raw in client.values()), name)
|
||||
|
||||
def test_the_call_reads_only_keys_the_client_sets(self):
|
||||
client = _module(self.resolve, "set_fact")["stalwart_jmap_client"]
|
||||
used = set(
|
||||
re.findall(
|
||||
r"stalwart_jmap_client\.(\w+)", self.call["cmd"] + self.call["stdin"]
|
||||
)
|
||||
)
|
||||
self.assertTrue(used)
|
||||
self.assertLessEqual(used, set(client))
|
||||
|
||||
def test_the_client_resolves_only_while_undefined(self):
|
||||
condition = "{{ (" + str(self.resolve.get("when", True)) + ") | bool }}"
|
||||
self.assertTrue(_render(condition))
|
||||
self.assertFalse(_render(condition, stalwart_jmap_client={"url": "u"}))
|
||||
|
||||
def test_the_client_resolution_is_masked(self):
|
||||
self.assertEqual(
|
||||
self.resolve["no_log"], "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
|
||||
)
|
||||
|
||||
def test_a_swarm_address_reaches_the_shell_unresolved(self):
|
||||
client = _resolved_client(
|
||||
STALWART_JMAP_URL="http://localhost:8080/jmap/",
|
||||
STALWART_ADMIN_USER="stalwartadmin",
|
||||
STALWART_ADMIN_PASSWORD=PASSWORD,
|
||||
)
|
||||
cmd = _render(
|
||||
self.call["cmd"],
|
||||
STALWART_CONTAINER_ADDRESS=SWARM_ADDRESS,
|
||||
stalwart_jmap_client=client,
|
||||
)
|
||||
self.assertTrue(cmd.startswith(f"container exec -i {SWARM_ADDRESS} curl "))
|
||||
self.assertTrue(cmd.endswith(" http://localhost:8080/jmap/"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user