Merge branch 'feature/updater-lockdown-probe' into feature/updater

This commit is contained in:
kevinveenbirkenbach committed 2026-10-29 12:14:57 +01:00
commit 3966abdb16
44 files changed
+1134 -94

No files matched your search

-2
View File
@@ -101,8 +101,6 @@ def placed_roles(roles_dir: Path) -> list[Path]:
def _fields(role: Path) -> dict[str, str]:
# Exception: O_NONBLOCK keeps a FIFO planted under the marker name from
# blocking the open; the fstat below refuses it.
try:
descriptor = os.open(
role / MARKER_FILE, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK
-4
View File
@@ -156,10 +156,6 @@ def _check_local(
def _git(cwd: str | None, arguments: list[str], failure: str) -> bytes:
# Exception: a git hook exports GIT_DIR, GIT_INDEX_FILE and friends, which
# would point the scratch repository at the core checkout; the askpass
# helpers and the detached session keep a fetch that needs credentials
# from prompting.
env = {key: value for key, value in os.environ.items() if key not in _DROPPED_ENV}
env.update(GIT_TERMINAL_PROMPT="0", SSH_ASKPASS_REQUIRE="never")
result = subprocess.run(
+18
View File
@@ -1,6 +1,7 @@
# nocheck: comments-valid explanatory WHY-comments in this SPOT lookup predate the stricter lint
from __future__ import annotations
from collections.abc import Mapping
from pathlib import Path
from typing import Any
@@ -11,6 +12,7 @@ from ansible.plugins.lookup import LookupBase
from plugins.lookup.applications import LookupModule as ApplicationsLookup
from plugins.lookup.domain import LookupModule as DomainLookup
from plugins.lookup.users import LookupModule as UsersLookup
from utils.cache.carrier import APPLICATIONS_RENDERED_FACT
from utils.domains.primary_domain import get_domain
from utils.mail.provider import deployed_roles, resolve_active_provider
from utils.roles.entity.name import get_entity_name
@@ -88,6 +90,19 @@ def _render(
return value
def _provider_carrier(variables: dict[str, Any], host: str) -> Mapping[str, Any] | None:
hostvars = variables.get("hostvars")
raw_get = getattr(hostvars, "raw_get", None)
try:
host_vars = raw_get(host) if callable(raw_get) else hostvars[host]
except (KeyError, TypeError):
return None
if not isinstance(host_vars, Mapping):
return None
carrier = host_vars.get(APPLICATIONS_RENDERED_FACT)
return carrier if isinstance(carrier, Mapping) else None
class LookupModule(LookupBase):
def run(
self,
@@ -313,6 +328,9 @@ class LookupModule(LookupBase):
provider_groups = fallback
provider_view = dict(variables)
provider_view["group_names"] = list(provider_groups or fallback)
carrier = _provider_carrier(variables, provider_hosts[0])
if carrier is not None:
provider_view[APPLICATIONS_RENDERED_FACT] = carrier
forwarded = {
k: v for k, v in getattr(self, "_kwargs", {}).items() if k == "roles_dir"
}
+1 -1
View File
@@ -13,7 +13,7 @@ ollama:
backup:
no_stop_required: true
image: ollama/ollama
version: 0.40.12
version: 0.40.13
name: ollama
min_storage: 3473MB
cpus: "4.0"
+8 -7
View File
@@ -25,12 +25,13 @@ Env (rendered into test.env from templates/test.env.j2):
ONION_HOST the node onion host
ONION_PORTS space-separated forwarded ports; empty means there is
nothing to check
ONION_PORT_TIMEOUT seconds per connect attempt. 60 for the same reason the
domain probe uses it: Tor's SocksTimeout default is 120s
and abandoning earlier is destructive, because the next
attempt opens a fresh SOCKS connection and throws away
the descriptor fetches and rendezvous retries the last
one was still working through
ONION_PORT_TIMEOUT seconds per connect attempt, above Tor's SocksTimeout
(120s by default; torrc does not set it). Every attempt
builds a fresh rendezvous, which Tor keeps working on
until SocksTimeout and then ends itself with reply 0x06
and a "Giving up" notice in its log. An attempt abandoned
earlier throws away a rendezvous Tor may still complete
and leaves Tor's log without a trace of the failure
ONION_PORT_RETRIES passes over the failing ports. Below the domain probe's
RETRIES because this phase multiplies by the port count,
and meta/tests.yml caps the whole CLI test
@@ -148,7 +149,7 @@ def main() -> int:
retries = int(_required("ONION_PORT_RETRIES", "passes over the failing ports"))
sleep_seconds = float(_required("SLEEP_SECONDS", "wait between passes"))
timeout = float(
_required("ONION_PORT_TIMEOUT", "seconds per connect, flavor-dependent")
_required("ONION_PORT_TIMEOUT", "seconds per connect, above Tor's SocksTimeout")
)
ports = [int(entry) for entry in os.environ.get("ONION_PORTS", "").split()]
+1 -1
View File
@@ -1,3 +1,3 @@
---
cli:
timeout: 7200
timeout: 9000
+22 -2
View File
@@ -80,7 +80,6 @@
dest: "{{ TOR_ENTRYPOINT_HOST }}"
mode: "0755"
become: true
register: _tor_entrypoint
- name: "🩺 Stage the Tor container healthcheck into the build context"
stack_host_copy:
@@ -88,7 +87,28 @@
dest: "{{ TOR_HEALTHCHECK_HOST }}"
mode: "0755"
become: true
register: _tor_healthcheck
- name: "🧩 Load the container CLI the Tor run probe calls"
include_role:
name: sys-svc-container
public: false
when: run_once_sys_svc_container is not defined
- name: "🔍 Check whether Tor already runs from an earlier deploy"
ansible.builtin.command: >-
{% if DEPLOYMENT_MODE == 'swarm' %}
container service ps -q --filter desired-state=running
{{ lookup('container_service', application_id, 'tor') }}
{% else %}
container ps -q --filter status=running --filter name=^{{ TOR_CONTAINER }}$
{% endif %}
register: _tor_running_before
changed_when: false
failed_when:
- _tor_running_before.rc != 0
- "'no such service' not in (_tor_running_before.stderr | lower)"
become: true
when: IS_STACK_HOST | bool
- name: "🧩 For '{{ application_id }}': Load sys-svc-compose"
include_role:
+1 -1
View File
@@ -2,7 +2,7 @@ NGINX_SERVERS_DIR={{ lookup('nginx', 'directories.configuration.servers') }}
ONION_HOST={{ lookup('config', 'svc-net-tor', 'services.tor.node') }}
ONION_PORTS={{ lookup('tor_ports') | map(attribute='onion_port') | unique | sort | join(' ') | dotenv_quote }}
ONION_PORT_RETRIES=2
ONION_PORT_TIMEOUT=60
ONION_PORT_TIMEOUT=130
RETRIES=5
SLEEP_SECONDS=10
TOR_CONTAINER={{ lookup('container_address', application_id, 'tor') }}
+3 -4
View File
@@ -60,7 +60,6 @@ TOR_HEALTHCHECK_HOST: "{{ [ TOR_INSTANCE_DIR, 'healthcheck.sh' ] | path_join }}"
TOR_ENTRYPOINT_PATH: "/usr/local/bin/tor-entrypoint.sh"
TOR_HEALTHCHECK_PATH: "/usr/local/bin/tor-healthcheck.sh"
TOR_RESTART_REQUIRED: >-
{{ (_tor_hs_keys is changed)
or (_tor_torrc is changed)
or (_tor_entrypoint is changed)
or (_tor_healthcheck is changed) }}
{{ ((_tor_hs_keys is changed) or (_tor_torrc is changed))
and (_tor_running_before is not skipped)
and (_tor_running_before.stdout | trim | length > 0) }}
+2 -1
View File
@@ -45,7 +45,8 @@
path: "{{ SVC_STORAGE_NFS_CLIENT_PROBE_DIR }}"
src: "{{ STORAGE_NFS_CLIENT_SRC }}"
fstype: "{{ STORAGE_NFS_FSTYPE }}"
opts: "{{ STORAGE_NFS_MOUNT_OPTS }},_netdev,noauto"
opts: "{{ STORAGE_NFS_MOUNT_OPTS }},_netdev"
boot: false
state: ephemeral
register: svc_storage_nfs_client_probe
failed_when:
+2 -1
View File
@@ -53,7 +53,8 @@ Solid `1:1` edges are fixed relationships; dashed `0..1` edges are conditional (
to every routing-mesh port no application declares under `ports.public`.
Loopback, the Docker bridges and the swarm nodes' own addresses stay
admitted. The table is loaded before a node joins and reloaded on every node
after the join and after each stack deploy.
after the join and after each stack deploy that changes the published ports
or the swarm nodes.
## Quick Setup
@@ -11,6 +11,11 @@
file: "{{ lookup('path_absolute', 'roles/svc-net-firewall/vars/main.yml') }}"
name: swarm_ingress_firewall
- name: "🧮 Swarm ingress guard | Fingerprint the bound the swarm nodes need"
ansible.builtin.set_fact:
swarm_ingress_guard_digest: "{{ [SWARM_INGRESS_GUARDED, SWARM_INGRESS_PEERS] | to_json | hash('sha256') }}"
run_once: true
- name: "🛡️ Swarm ingress guard | Reload the bound on every swarm node"
ansible.builtin.include_tasks:
file: "{{ lookup('path_absolute', 'roles/svc-net-firewall/tasks/02_fragment.yml') }}"
@@ -28,3 +33,14 @@
loop_control:
loop_var: swarm_ingress_guard_node
run_once: true
when: (hostvars[swarm_ingress_guard_node].swarm_ingress_guard_loaded | default('')) != swarm_ingress_guard_digest
- name: "🧾 Swarm ingress guard | Record the bound every swarm node now carries"
ansible.builtin.set_fact:
swarm_ingress_guard_loaded: "{{ swarm_ingress_guard_digest }}"
delegate_to: "{{ swarm_ingress_guard_node }}"
delegate_facts: true
loop: "{{ groups['svc-swarm-node'] }}"
loop_control:
loop_var: swarm_ingress_guard_node
run_once: true
@@ -24,14 +24,24 @@
ansible.builtin.set_fact:
service_loader_post_run: "{{ (service_loader_post_run | default([])) + list_or_shoot_roles }}"
changed_when: false
run_once: false
when: (service_loader_preloading | default(false) | bool)
- name: "🎯 Shoot the post-load role directly (no preload context active)"
ansible.builtin.include_role:
name: "{{ list_or_shoot_roles | first }}"
vars:
application_id: "{{ list_or_shoot_roles | first }}"
when:
- not (service_loader_preloading | default(false) | bool)
- list_or_shoot_roles | length == 1
- name: "🎯 Shoot post-load roles directly (no preload context active)"
ansible.builtin.include_role:
name: "{{ list_or_shoot_role }}"
vars:
application_id: "{{ list_or_shoot_role }}"
loop: "{{ list_or_shoot_roles }}"
loop: "{{ list_or_shoot_roles if list_or_shoot_roles | length > 1 else [] }}"
loop_control:
loop_var: list_or_shoot_role
when: not (service_loader_preloading | default(false) | bool)
+5
View File
@@ -55,6 +55,11 @@
loop_var: service_item
label: "{{ service_item.id }}"
- name: "📂 Close the preload context before the post-load flush"
ansible.builtin.set_fact:
service_loader_preloading: false
changed_when: false
- name: "▶️ Run service-loader post-load roles"
ansible.builtin.include_role:
name: "{{ post_run_role }}"
@@ -1,6 +1,8 @@
---
- name: "🔍 Probe the vhosts this play queued"
when: front_proxy_deferred_probes | length > 0
when:
- front_proxy_deferred_probes | length > 0
- inventory_hostname == groups['svc-prx-openresty'][0]
block:
- name: "🔍 Check if the queued vhosts are reachable"
uri_probe_batch:
+1
View File
@@ -55,6 +55,7 @@
url: "{{ lookup('tls', front_proxy_domain, 'url.base') }}"
status_codes: "{{ lookup('applications') | accepted_status_codes(front_proxy_application_id, lookup('tls', front_proxy_domain, 'domain')) }}"
changed_when: false
when: IS_STACK_HOST | bool
- name: 📂 Include dependencies
include_role:
+1 -4
View File
@@ -2,15 +2,12 @@
- include_tasks: 00_core.yml
when: run_once_sys_svc_compose is not defined
- name: "🧭 Route docker cleanup/health deps through service-loader"
- name: "🧭 Route the compose hard repair through service-loader"
ansible.builtin.include_role:
name: sys-service-loader
tasks_from: list_or_shoot.yml
vars:
list_or_shoot_roles:
- sys-ctl-cln-docker
- sys-ctl-hlth-container
- sys-ctl-hlth-volumes
- sys-ctl-rpr-container-hard
- name: "🧭 Route svc-bkp-volume-2-local through service-loader (when deployed)"
+10
View File
@@ -94,4 +94,14 @@
include_tasks: "03_reset.yml"
when: MODE_RESET | bool
- name: "🧭 Route docker cleanup/health deps through service-loader"
ansible.builtin.include_role:
name: sys-service-loader
tasks_from: list_or_shoot.yml
vars:
list_or_shoot_roles:
- sys-ctl-cln-docker
- sys-ctl-hlth-container
- sys-ctl-hlth-volumes
- include_tasks: utils/once/flag.yml
@@ -1,4 +1,7 @@
---
# REQUIRED:
# sys_svc_container_prepull_extra # image references to pull besides the app images of the deploy, e.g. the E2E runner
- name: "📥 Pre-pull the images of this deploy in the background"
when:
- not (STORAGE_CONSTRAINED | bool)
@@ -8,10 +11,9 @@
- name: "🧮 Derive the images the deploy of the host runs"
ansible.builtin.set_fact:
sys_svc_container_prepull_plan: >-
{{
lookup('applications')
| update_pull_images(lookup('application_closure', group_names | intersect(lookup('deployment').running)))
}}
{%- set plan = lookup('applications')
| update_pull_images(lookup('application_closure', group_names | intersect(lookup('deployment').running))) -%}
{{ plan | combine({'pull': plan.pull + (sys_svc_container_prepull_extra | reject('in', plan.pull) | list)}) }}
- name: "📥 Pull the images the host does not hold yet"
ansible.builtin.include_tasks: "{{ role_path }}/tasks/utils/prepull/pull.yml"
+5 -3
View File
@@ -42,12 +42,14 @@
block:
- name: 🚀 Run CLI test for each detected role
ansible.builtin.include_tasks: run_one.yml
vars:
application_id: "{{ cli_role_id }}"
loop: "{{ _cli_apps }}"
loop_control:
loop_var: application_id # nocheck: loop-var-shadow the item IS the role id, so run_one reads the same value the tested role declares
label: "{{ application_id }}"
loop_var: cli_role_id
label: "{{ cli_role_id }}"
when: >-
not (lookup('role_tests', application_id, 'cli.controller_only', default=false) | bool)
not (lookup('role_tests', cli_role_id, 'cli.controller_only', default=false) | bool)
or (ansible_connection | default('')) == 'local'
- name: ✅ Fail if there were CLI test failures
@@ -16,10 +16,12 @@
- name: ▶️ Run Playwright for each detected role
ansible.builtin.include_tasks: 02_run_one.yml
vars:
application_id: "{{ playwright_role_id }}"
loop: "{{ TEST_E2E_PLAYWRIGHT_APPS }}"
loop_control:
loop_var: application_id # nocheck: loop-var-shadow the item IS the role id, so 02_run_one reads the same value the tested role's own vars declare
label: "{{ application_id }}"
loop_var: playwright_role_id
label: "{{ playwright_role_id }}"
- name: 🔁 Stash first-pass failures for one flake retry
ansible.builtin.set_fact:
@@ -29,10 +31,12 @@
- name: 🔁 Retry failed roles once on rotated Tor circuits
ansible.builtin.include_tasks: 02_run_one.yml
vars:
application_id: "{{ playwright_role_id }}"
loop: "{{ _playwright_retry_roles | default([]) }}"
loop_control:
loop_var: application_id # nocheck: loop-var-shadow same loop body, same identity between item and role id
label: "{{ application_id }} (retry)"
loop_var: playwright_role_id
label: "{{ playwright_role_id }} (retry)"
- name: 🛑 Fail if there were Playwright failures
ansible.builtin.fail:
@@ -234,10 +234,6 @@ async function pdsAdminUpdatePassword(did, newPassword) {
// --- High-level orchestration --------------------------------------
// Exception: the account map lives in the broker's own volume, never in a
// Keycloak user attribute; a bluesky_* attribute write under LDAP federation
// in WRITABLE mode triggers a schema-violating sync push that breaks later
// OIDC logins of that user.
const store = openStore(CONFIG.storePath);
class HandleChoiceRequired extends Error {}
+1 -1
View File
@@ -62,7 +62,7 @@ web:
view:
enabled: false
repository: https://github.com/bluesky-social/social-app.git
ref: "1.136.0"
ref: "1.137.0"
ports:
local:
http: 8051
+1 -1
View File
@@ -1,4 +1,4 @@
# To-dos
- Move off PHP 7.4. The pinned `nfrastack/fusiondirectory:1.5` image builds on `ghcr.io/nfrastack/container-nginx-php-fpm` with PHP 7.4 on Alpine 3.15, which is end of life and gets no fix for CVE-2026-109618 and CVE-2026-109619.
- Mount the shared PHP hardening ini (`FILE_PHP_HARDENING_INI`) once the image's scan dir is verified. The nfrastack init scripts manage `conf.d` at runtime, so a read-only mount there has to be tested before it ships.
- No FusionDirectory release runs on a supported PHP yet: upstream certifies 1.4 to 1.6 for PHP 7.4 only, `nfrastack/container-fusiondirectory` builds its image only on PHP 7.4 and Alpine 3.15, and the PHP 8 port lives on upstream's unreleased `php8` branch. Until then the shared hardening ini mounted at `/etc/php7/conf.d/zz-infinito-hardening.ini` carries the mitigations for both CVEs. Options: rebuild the image on upstream's first `php8` release, or build `nfrastack/container-fusiondirectory` against a pinned `php8` commit on `container-nginx-php-fpm` 8.4 before that release.
@@ -1,4 +1,11 @@
---
php_hardening:
type: config
source: "{{ FILE_PHP_HARDENING_INI }}"
mode: "0444"
mounts:
- service: fusiondirectory
target: /etc/php7/conf.d/zz-infinito-hardening.ini
# nocheck: embedded-state-on-nfs generated asset cache; the authoritative state lives in LDAP
data:
type: volume
@@ -11,6 +11,6 @@ server {
{% include 'roles/sys-svc-proxy/templates/location/upload.conf.j2' %}
{% set container_name = MATRIX_SYNAPSE_NAME if matrix_flavor == 'compose' else MATRIX_MDAD_CONTAINER %}
{% set container_name = MATRIX_SYNAPSE_SERVICE if matrix_flavor == 'compose' else lookup('container_service', application_id, MATRIX_MDAD_SERVICE_KEY) %}
{% include 'roles/web-app-prometheus/templates/nginx/locations.conf.j2' %}
}
@@ -32,6 +32,10 @@
end
metric_stack_up:set(up, {ngx.var.host, ngx.var.app_id})
if metric_docker_health_entries then
local health = ngx.shared.health_containers
metric_docker_health_entries:set(health and #health:get_keys(0) or 0)
end
prometheus:collect()
}
}
@@ -89,6 +89,41 @@ test("metricz endpoint is accessible and returns prometheus text format", async
)
.toBe(true);
}
const metriczHost = new URL(metriczUrl).hostname.toLowerCase();
const targetIds = new Set(prometheusTargetRoles.map((target) => target.id));
await expect
.poll(
async () => {
const scrape = await apiGetOnion(request, metriczUrl, { timeout: resolveTimeout(30_000) });
const text = await scrape.text();
const entries = text.match(/^nginx_docker_health_entries (\S+)$/m);
const stackUp = [...text.matchAll(/^nginx_stack_up\{host="([^"]*)",app="([^"]*)"\} (\S+)$/gm)];
const failures = [];
if (!entries) {
failures.push("nginx_docker_health_entries is missing");
} else if (!(Number(entries[1]) > 0)) {
failures.push(`nginx_docker_health_entries is ${entries[1]}`);
}
if (!stackUp.some(([, host]) => host === metriczHost)) {
failures.push(`nginx_stack_up{host="${metriczHost}"} is missing`);
}
for (const [, host, app, value] of stackUp) {
if ((host === metriczHost || targetIds.has(app)) && value !== "1") {
failures.push(`nginx_stack_up{host="${host}",app="${app}"} is ${value}`);
}
}
return failures;
},
{
timeout: resolveTimeout(60_000),
message:
"/metricz must show a populated docker-health dict (nginx_docker_health_entries > 0) " +
"and nginx_stack_up 1 for the scraped vhost and every consumer role that has a series. " +
"An entries count of 0 means the OpenResty privileged agent cannot read /var/run/docker.sock.",
},
)
.toEqual([]);
});
// Scenario II: direct goto Prometheus → SSO login (as admin) → verify Prometheus UI → logout
+1 -1
View File
@@ -71,7 +71,7 @@
stack_host_template:
src: "roles/web-app-prometheus/templates/nginx/prometheus.conf.j2"
dest: "{{ [ lookup('nginx','directories.configuration.global'), 'prometheus.conf' ] | path_join }}"
notify: restart-openresty
notify: reload-openresty
- name: "🧱 Rebuild the MCP adapter of {{ application_id }}"
ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/svc-ai-mcp-adapter/tasks/rebuild.yml') }}"
@@ -12,7 +12,9 @@
service registry.
$app_id — Ansible role ID attached as Lua metrics label.
$container_name — Docker container name for /healthz/live polling.
$container_name — health_containers key for /healthz/live polling: the
container name in compose, the swarm service name
(lookup('container_service')) in swarm.
Callers may set {% set container_name = "..." %} before
this include to override the default (derived from the
canonical service key via get_entity_name).
@@ -27,7 +29,7 @@
set $container_name "{{ container_name }}";
{% else %}
{% set prometheus_service_key = application_id | get_entity_name %}
set $container_name "{{ lookup('config', application_id, 'services.' + prometheus_service_key + '.name', '') }}";
set $container_name "{{ lookup('container_service', application_id, prometheus_service_key) if lookup('config', application_id, 'services.' + prometheus_service_key + '.name', '') else '' }}";
{% endif %}
{% include 'roles/web-app-prometheus/templates/nginx/healthz.conf.j2' %}
{% include 'roles/web-app-prometheus/templates/nginx/location.conf.j2' %}
@@ -4,6 +4,13 @@
lua_shared_dict health_containers 1m;
init_by_lua_block {
local ok, err = require("ngx.process").enable_privileged_agent()
if not ok then
ngx.log(ngx.ERR, "docker-health: cannot enable the privileged agent: ", err)
end
}
init_worker_by_lua_block {
-- lua-resty-prometheus (master) requires init() in init_worker_by_lua_block,
-- not init_by_lua_block, so that each worker initialises its own state.
@@ -34,18 +41,17 @@
"Overall stack availability per vhost (1=up, 0=down)",
{"host", "app"}
)
metric_docker_health_entries = prometheus:gauge(
"nginx_docker_health_entries",
"Unexpired health_containers entries written by the docker-health poller"
)
metric_tls_handshakes = prometheus:counter(
"nginx_tls_handshakes_total",
"Total TLS handshakes by protocol",
{"host", "app", "protocol"}
)
-- Docker health polling — worker 0 only to avoid duplicate socket traffic.
-- Queries GET /v1.41/containers/json via the Docker Unix socket, then
-- stores each container's liveness in health_containers. /healthz/live
-- reads the cache synchronously so Docker state is checked on every probe
-- without adding socket-connect latency to the request path.
if ngx.worker.id and ngx.worker.id() == 0 then
if require("ngx.process").type() == "privileged agent" then
local function poll_docker_health(premature)
if premature then return end
local cjson = require("cjson.safe")
@@ -81,6 +87,7 @@
local containers = cjson.decode(body)
if type(containers) ~= "table" then return end
local services = {}
for _, c in ipairs(containers) do
local names = c.Names
if type(names) == "table" and names[1] then
@@ -95,8 +102,17 @@
local is_up = (state == "running") and (health_tag ~= "unhealthy")
-- TTL 60 s — entries expire if the container disappears between polls
health:set(name, is_up and "up" or "down", 60)
local labels = c.Labels
local service = type(labels) == "table" and labels["com.docker.swarm.service.name"]
if type(service) == "string" and service ~= "" then
services[service] = services[service] or is_up
end
end
end
for service, up in pairs(services) do
health:set(service, up and "up" or "down", 60)
end
end
-- Seed immediately so the dict is populated before the first request
@@ -33,12 +33,14 @@ test("administrator: stalwart recovery admin native login and logout", async ({
.first()
.click();
// Stalwart asks consent for every registered OAuth client before it shows the password form.
// Stalwart v1.0.0 asks consent for the OAuth client before the password form; v1.0.1 shows the password form directly.
const allow = page.getByRole("button", { name: /^allow$/i });
await allow.waitFor({ state: "visible", timeout: resolveTimeout(30_000) });
await allow.click();
const passwordField = page.locator("input[type='password'], input[name='password']").first();
await allow.or(passwordField).first().waitFor({ state: "visible", timeout: resolveTimeout(30_000) });
if (await allow.isVisible()) {
await allow.click();
}
await passwordField.waitFor({ state: "visible", timeout: resolveTimeout(30_000) });
await passwordField.fill(stalwartAdminPassword);
await page
@@ -51,9 +53,15 @@ test("administrator: stalwart recovery admin native login and logout", async ({
page.getByText(/invalid username or password/i)
).toBeHidden({ timeout: resolveTimeout(15_000) });
await expect(
page.locator("nav, .sidebar, [class*='menu'], h1, h2").filter({ hasText: /dashboard|domains|account|settings|directory/i }).first()
).toBeVisible({ timeout: resolveTimeout(30_000) });
const dashboard = page
.locator("nav, .sidebar, [class*='menu'], h1, h2")
.filter({ hasText: /dashboard|domains|account|settings|directory/i })
.first();
await allow.or(dashboard).first().waitFor({ state: "visible", timeout: resolveTimeout(30_000) });
if (await allow.isVisible()) {
await allow.click();
}
await expect(dashboard).toBeVisible({ timeout: resolveTimeout(30_000) });
if (oidcIssuerUrl) {
expect(page.url()).not.toContain(oidcIssuerUrl);
+10 -2
View File
@@ -2,15 +2,23 @@
# Reusable management JMAP call: in `stalwart_jmap_body` (JSON), out `stalwart_jmap`.
# Param: stalwart_jmap_body - JSON request; a batch carries one method call per object owner, its call id naming that owner in a refusal.
# Param: stalwart_jmap_tolerate_rejected - true when the caller reads a set-level rejection itself (e.g. primaryKeyViolation).
- name: "🔧 Resolve the Stalwart management URL and admin login once per run"
ansible.builtin.set_fact:
stalwart_jmap_client:
url: "{{ STALWART_JMAP_URL }}"
user_password: "{{ STALWART_ADMIN_USER ~ ':' ~ STALWART_ADMIN_PASSWORD }}"
when: stalwart_jmap_client is not defined
no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
- name: "🔧 Stalwart JMAP call"
ansible.builtin.shell:
cmd: >-
container exec -i {{ STALWART_CONTAINER_ADDRESS }}
{{ 30 | curl }} -sS -f -K -
-H "Content-Type:application/json"
{{ STALWART_JMAP_URL }}
{{ stalwart_jmap_client.url }}
stdin: |-
user = {{ (STALWART_ADMIN_USER ~ ':' ~ STALWART_ADMIN_PASSWORD) | to_json(ensure_ascii=False) }}
user = {{ stalwart_jmap_client.user_password | to_json(ensure_ascii=False) }}
data-binary = {{ stalwart_jmap_body | to_json(ensure_ascii=False) }}
register: stalwart_jmap_raw
changed_when: false
@@ -15,7 +15,6 @@ define('YOURLS_UPGRADING', true);
$infinito_upgrade_done = false;
// Exception: YOURLS reports bootstrap and upgrade errors with die(), which exits 0; only this handler turns them into a failure.
register_shutdown_function(static function (): void {
if ($GLOBALS['infinito_upgrade_done']) {
return;
+13 -13
View File
@@ -53,7 +53,7 @@ git worktree add "${path}" "${branch}"
git_common_dir="$(git rev-parse --path-format=absolute --git-common-dir)"
lab_octet="$((244 + slot))"
lab_prefix="10.160.${slot}"
echo ">>> Pinning slot ${slot} in ${path}/custom.env"
cat >"${path}/custom.env" <<PINS
@@ -64,12 +64,12 @@ INFINITO_GATEWAY=172.30.${slot}.1
INFINITO_DNS_IP=172.30.${slot}.53
INFINITO_IP4=172.30.${slot}.10
INFINITO_BIND_IP=127.0.0.$((slot + 1))
INFINITO_SWARM_LAB_SUBNET=192.168.${lab_octet}.0/24
INFINITO_SWARM_MGR_IP=192.168.${lab_octet}.10
INFINITO_SWARM_WRK1_IP=192.168.${lab_octet}.11
INFINITO_SWARM_WRK2_IP=192.168.${lab_octet}.12
INFINITO_SWARM_NFS_IP=192.168.${lab_octet}.13
INFINITO_SWARM_BACKUP_IP=192.168.${lab_octet}.14
INFINITO_SWARM_LAB_SUBNET=${lab_prefix}.0/24
INFINITO_SWARM_MGR_IP=${lab_prefix}.10
INFINITO_SWARM_WRK1_IP=${lab_prefix}.11
INFINITO_SWARM_WRK2_IP=${lab_prefix}.12
INFINITO_SWARM_NFS_IP=${lab_prefix}.13
INFINITO_SWARM_BACKUP_IP=${lab_prefix}.14
INFINITO_RUNNER_PREFIX=infinito-${slug}
INFINITO_CONTAINER=infinito_nexus_${slug//[.-]/_}
INFINITO_CACHE_NETWORK=${cache_network}
@@ -86,12 +86,12 @@ env -i -C "${path}" \
INFINITO_DNS_IP="172.30.${slot}.53" \
INFINITO_IP4="172.30.${slot}.10" \
INFINITO_BIND_IP="127.0.0.$((slot + 1))" \
INFINITO_SWARM_LAB_SUBNET="192.168.${lab_octet}.0/24" \
INFINITO_SWARM_MGR_IP="192.168.${lab_octet}.10" \
INFINITO_SWARM_WRK1_IP="192.168.${lab_octet}.11" \
INFINITO_SWARM_WRK2_IP="192.168.${lab_octet}.12" \
INFINITO_SWARM_NFS_IP="192.168.${lab_octet}.13" \
INFINITO_SWARM_BACKUP_IP="192.168.${lab_octet}.14" \
INFINITO_SWARM_LAB_SUBNET="${lab_prefix}.0/24" \
INFINITO_SWARM_MGR_IP="${lab_prefix}.10" \
INFINITO_SWARM_WRK1_IP="${lab_prefix}.11" \
INFINITO_SWARM_WRK2_IP="${lab_prefix}.12" \
INFINITO_SWARM_NFS_IP="${lab_prefix}.13" \
INFINITO_SWARM_BACKUP_IP="${lab_prefix}.14" \
INFINITO_RUNNER_PREFIX="infinito-${slug}" \
INFINITO_CACHE_NETWORK="${cache_network}" \
python3 -m cli.meta.env
-2
View File
@@ -101,8 +101,6 @@ fi
cmd+=(--cache-server-addr 127.0.0.1)
cmd+=(--artifact-server-addr 127.0.0.1)
if [[ "${ACT_BIND}" != "true" ]] && compgen -G "${_act_repo_root}/roles/web-app-cst-*" >/dev/null; then
# Exception: act copies the checkout through .gitignore, which drops the
# pulled custom roles, so the job binds the checkout to keep them.
echo "act: binding the checkout to carry the pulled custom roles (roles/web-app-cst-*)"
ACT_BIND=true
fi
@@ -97,19 +97,24 @@ if [ -n "${CACHE_NET}" ]; then
cache_prefix="${cache_subnet%.0/24}"
node_octet=$((200 + INFINITO_INSTANCE * 5))
pin_nodes=true
if [ "$((node_octet + 4))" -gt 254 ]; then
echo "FAILURE: INFINITO_INSTANCE=${INFINITO_INSTANCE} pushes the node band past ${cache_prefix}.254" >&2
exit 1
echo "==> INFINITO_INSTANCE=${INFINITO_INSTANCE} leaves no pinned node band in ${cache_prefix}.0/24; Docker assigns the node addresses from ${CACHE_NET}'s pool" >&2
pin_nodes=false
fi
for node in "${MGR}" "${WRK1}" "${WRK2}" "${NFS_SERVER}" "${BACKUP_NODE}"; do
if ! connect_err="$(docker network connect --ip "${cache_prefix}.${node_octet}" "${CACHE_NET}" "${node}" 2>&1)"; then
connect_args=()
if [ "${pin_nodes}" = true ]; then
connect_args=(--ip "${cache_prefix}.${node_octet}")
fi
if ! connect_err="$(docker network connect "${connect_args[@]}" "${CACHE_NET}" "${node}" 2>&1)"; then
case "${connect_err}" in
*"already exists in network"* | *"already attached to network"*)
echo "==> ${node} already on ${CACHE_NET}; keeping its current address" >&2
;;
*)
echo "FAILURE: cannot pin ${node} to ${cache_prefix}.${node_octet} on ${CACHE_NET}: ${connect_err}" >&2
echo "FAILURE: cannot attach ${node} to ${CACHE_NET} (${connect_args[*]:-pool address}): ${connect_err}" >&2
exit 1
;;
esac
+9 -5
View File
@@ -191,17 +191,17 @@
}}
when: _package_batch_ids | length > 0
- name: "🥾 Bootstrap svc-swarm-node"
ansible.builtin.include_role:
name: svc-swarm-node
when: "'svc-swarm-node' | application_allowed(group_names, lookup('deployment').whitelist)"
- name: "📂 Open service-loader preload context BEFORE Bootstrap"
ansible.builtin.set_fact:
service_loader_post_run: []
service_loader_preloading: true
changed_when: false
- name: "🥾 Bootstrap svc-swarm-node"
ansible.builtin.include_role:
name: svc-swarm-node
when: "'svc-swarm-node' | application_allowed(group_names, lookup('deployment').whitelist)"
- name: "🥾 Bootstrap NFS clients + registry + sandbox runtimes (MUST precede Preload)"
ansible.builtin.include_tasks: ./tasks/utils/setup/group.yml
loop:
@@ -224,6 +224,10 @@
ansible.builtin.include_role:
name: sys-svc-container
tasks_from: utils/prepull/start.yml
vars:
sys_svc_container_prepull_extra: >-
{{ [lookup('config', 'test-e2e-playwright', 'services.playwright.image') ~ ':' ~ lookup('config', 'test-e2e-playwright', 'services.playwright.version')]
if RUNTIME in ['dev', 'act', 'github'] and 'server' in (group_names | get_deployment_types_from_groups) else [] }}
- name: 🛠️ Preload shared services
ansible.builtin.include_role:
+1
View File
@@ -18,3 +18,4 @@
- name: "🧊 Carry the applications SPOT past nested {{ load_app_id }}"
applications_carrier: {}
run_once: false
@@ -1,16 +1,29 @@
import glob
import json
import re
import shutil
import subprocess
import tempfile
import unittest
from collections.abc import Mapping
from pathlib import Path
import yaml
from ansible.parsing.dataloader import DataLoader
from ansible.template import Templar, trust_as_template
from utils.cache.applications import get_application_defaults
from utils.cache.files import read_text
from utils.cache.yaml import load_yaml_any
from utils.roles.applications.config import get as get_app_conf
from utils.roles.applications.services.registry import is_explicit_truth
from utils.roles.entity.name import get_entity_name
from utils.roles.mapping import (
ROLE_FILE_DEFAULTS_MAIN,
ROLE_FILE_META_SERVICES,
ROLE_FILE_META_VOLUMES,
ROLE_FILE_TEMPL_COMPOSE,
ROLE_FILE_VARS_MAIN,
)
from . import PROJECT_ROOT
@@ -18,11 +31,452 @@ from . import PROJECT_ROOT
PROMETHEUS_APP_ID = "web-app-prometheus"
ROLES_DIR = PROJECT_ROOT / "roles"
_DOMAIN = "infinito.test"
_MODES = ("compose", "swarm")
_GROUP_NAMES = (
"web-app-prometheus",
"web-app-gitea",
"web-app-matrix",
"svc-prx-openresty",
)
_INJ_OFF = {
"logout": False,
"dashboard": False,
"css": False,
"matomo": False,
"javascript": False,
}
_CONTAINER_NAME_SET = re.compile(
r'^\s*set \$container_name "([^"]*)";\s*$', re.MULTILINE
)
_LUA_BLOCK_OPEN = re.compile(r"\b([a-z_]+_by_lua_block)\s*\{")
_DOCKER_SOCKET = "unix:/var/run/docker.sock"
_DOCKER_PS = {
"compose": [
{
"Names": ["/gitea"],
"State": "running",
"Status": "Up 3 minutes (unhealthy)",
"Labels": {"com.docker.compose.service": "gitea"},
},
{
"Names": ["/prometheus"],
"State": "running",
"Status": "Up 2 hours (healthy)",
"Labels": {"com.docker.compose.service": "prometheus"},
},
{
"Names": ["/matrix-synapse"],
"State": "running",
"Status": "Up 5 hours",
"Labels": None,
},
],
"swarm": [
{
"Names": ["/gitea_gitea.1.k3j2x"],
"State": "running",
"Status": "Up 3 minutes (unhealthy)",
"Labels": {
"com.docker.stack.namespace": "gitea",
"com.docker.swarm.service.name": "gitea_gitea",
},
},
{
"Names": ["/prometheus_prometheus.1.x9q2"],
"State": "running",
"Status": "Up 2 hours (healthy)",
"Labels": {"com.docker.swarm.service.name": "prometheus_prometheus"},
},
{
"Names": ["/matrix_synapse.1.old"],
"State": "running",
"Status": "Up 1 hour (unhealthy)",
"Labels": {"com.docker.swarm.service.name": "matrix_synapse"},
},
{
"Names": ["/matrix_synapse.2.new"],
"State": "running",
"Status": "Up 1 hour (healthy)",
"Labels": {"com.docker.swarm.service.name": "matrix_synapse"},
},
],
}
_LUA_HARNESS = r"""
local dir = arg[1]
local null = newproxy(false)
local function read(name)
local f = assert(io.open(dir .. "/" .. name))
local s = f:read("*a")
f:close()
return s
end
local body = read("containers.json")
local decoded = assert(loadfile(dir .. "/containers.lua"))(null)
local function world(ptype, connect_err)
local w = {connects = 0, warns = 0, now = 0, at = {}, every = {}, store = {}, ttl = {}, expires = {}, gauges = {}}
local function live(k) return w.store[k] ~= nil and (w.expires[k] == 0 or w.expires[k] > w.now) end
local function metric(name)
return {
set = function(_, v) w.gauges[name] = v end,
inc = function() end,
observe = function() end,
}
end
local prom = {
counter = function(_, n) return metric(n) end,
histogram = function(_, n) return metric(n) end,
gauge = function(_, n) return metric(n) end,
collect = function() w.collected = true end,
}
local modules = {
["ngx.process"] = {
type = function() return ptype end,
enable_privileged_agent = function() w.enabled = true return true end,
},
["resty.prometheus"] = {init = function() return prom end},
["cjson.safe"] = {decode = function(s) if s == body then return decoded end end},
}
local ngx = {
ERR = 4, WARN = 5, HTTP_OK = 200, HTTP_SERVICE_UNAVAILABLE = 503,
log = function(level) if level == 5 then w.warns = w.warns + 1 end end,
header = {},
var = {},
shared = {health_containers = {
set = function(_, k, v, ttl)
w.store[k] = v w.ttl[k] = ttl w.expires[k] = ttl == 0 and 0 or w.now + ttl
return true
end,
get = function(_, k) if live(k) then return w.store[k] end end,
get_keys = function()
local keys = {}
for k in pairs(w.store) do if live(k) then keys[#keys + 1] = k end end
return keys
end,
}},
timer = {
at = function(delay, fn) w.at[#w.at + 1] = delay fn(false) return true end,
every = function(delay) w.every[#w.every + 1] = delay return true end,
},
socket = {tcp = function()
local lines = {"HTTP/1.0 200 OK", "Api-Version: 1.41", "Content-Type: application/json", ""}
local n = 0
return {
connect = function(_, target)
w.connects = w.connects + 1 w.target = target
if connect_err then return nil, connect_err end
return true
end,
send = function(_, data) w.request = data return #data end,
receive = function(_, pattern)
if pattern == "*l" then n = n + 1 return lines[n] end
return body
end,
close = function() end,
}
end},
location = {capture = function() return {status = w.probe} end},
worker = {id = function() if ptype == "worker" then return 0 end end},
}
ngx.print = function() end
w.ngx = ngx
w.env = setmetatable({ngx = ngx, require = function(name) return assert(modules[name], name) end}, {__index = _G})
return w
end
local function run(w, name)
local chunk = assert(loadstring(read(name), name))
setfenv(chunk, w.env)
chunk()
end
local master = world("master")
run(master, "init.lua")
print("enabled\t" .. tostring(master.enabled))
local worker = world("worker")
run(worker, "init_worker.lua")
print("worker\t" .. worker.connects .. "/" .. #worker.at .. "/" .. #worker.every)
local agent = world("privileged agent")
run(agent, "init_worker.lua")
print("agent\t" .. tostring(agent.target) .. "\t" .. tostring(agent.request and agent.request:match("^[^\r\n]*")) .. "\t" .. table.concat(agent.at, ",") .. "/" .. table.concat(agent.every, ","))
for key, value in pairs(agent.store) do
print("dict\t" .. key .. "\t" .. value .. "/" .. agent.ttl[key])
end
local function request(w, key)
w.ngx.var = {container_name = key, host = "h", app_id = "a"}
w.probe = 200
w.ngx.status = nil
run(w, "live.lua")
run(w, "metricz.lua")
return tostring(w.ngx.status), tostring(w.gauges.nginx_stack_up), tostring(w.gauges.nginx_docker_health_entries)
end
for i = 2, #arg do
local status, up, entries = request(agent, arg[i])
print("live\t" .. arg[i] .. "\t" .. status)
print("metricz\t" .. arg[i] .. "\t" .. up .. "\t" .. entries)
end
agent.now = 61
local _, _, expired = request(agent, arg[2])
print("expired\t" .. expired)
local denied = world("privileged agent", "permission denied")
run(denied, "init_worker.lua")
print("denied\t" .. denied.connects .. "\t" .. denied.warns)
for i = 2, #arg do
local status, up, entries = request(denied, arg[i])
print("denied_metricz\t" .. arg[i] .. "\t" .. status .. "\t" .. up .. "\t" .. entries)
end
local stale = world("worker")
run(stale, "init_worker.lua")
stale.env.metric_docker_health_entries = nil
local stale_ok = pcall(request, stale, arg[2])
print("stale_conf\t" .. tostring(stale_ok) .. "\t" .. tostring(stale.collected))
"""
def _load_config(file_path: str) -> dict:
return load_yaml_any(file_path) or {}
def _trusted(value: object) -> object:
if isinstance(value, str):
return trust_as_template(value)
if isinstance(value, Mapping):
return {k: _trusted(v) for k, v in value.items()}
if isinstance(value, list):
return [_trusted(v) for v in value]
return value
def _vars_layer(paths: list[Path]) -> dict:
loaded: dict = {}
for path in paths:
if not path.is_file():
continue
data = load_yaml_any(str(path), default_if_missing={})
if isinstance(data, Mapping):
loaded.update({k: _trusted(v) for k, v in data.items()})
return loaded
def _render(application_id: str, template: str, mode: str, extra: dict) -> str:
"""Render a template through Ansible's templar with the deploy's own lookups.
Args:
application_id: role whose defaults and vars layer over group_vars/all.
template: template path relative to the repository root.
mode: DEPLOYMENT_MODE, ``compose`` or ``swarm``.
extra: variables the including task passes to the template.
Returns:
The rendered text.
"""
source = (PROJECT_ROOT / template).resolve()
return _template(
application_id, mode, extra, f"{{{{ lookup('template', '{source}') }}}}"
)
def _template(application_id: str, mode: str, extra: dict, expression: str) -> str:
"""Evaluate a Jinja expression with the variables a role deploy carries.
Args:
application_id: role whose defaults and vars layer over group_vars/all.
mode: DEPLOYMENT_MODE, ``compose`` or ``swarm``.
extra: variables layered on top.
expression: the Jinja source to evaluate.
Returns:
The evaluated text.
"""
applications = get_application_defaults(roles_dir=ROLES_DIR)
variables = {
**_vars_layer(sorted((PROJECT_ROOT / "group_vars" / "all").glob("*.yml"))),
**_vars_layer(
[
ROLES_DIR / application_id / ROLE_FILE_DEFAULTS_MAIN,
ROLES_DIR / application_id / ROLE_FILE_VARS_MAIN,
]
),
"applications": applications,
"application_id": application_id,
"group_names": list(_GROUP_NAMES),
"inventory_hostname": "localhost",
"playbook_dir": str(PROJECT_ROOT),
"ansible_search_path": [str(PROJECT_ROOT)],
"DEPLOYMENT_MODE": mode,
"TIMEOUT_FACTOR": 1,
"PRIMARY_DOMAIN": _DOMAIN,
"DOMAIN_PRIMARY": _DOMAIN,
"SYSTEM_EMAIL_DOMAIN": _DOMAIN,
"inj_enabled": _INJ_OFF,
**extra,
}
templar = Templar(loader=DataLoader(), variables=variables)
return str(templar.template(trust_as_template(expression)))
def _vhost(application_id: str, service_key: str, mode: str, **extra) -> str:
"""Render the vhost include that sets ``$container_name`` for one application.
Args:
application_id: the proxied application.
service_key: the ``services.<key>`` entry whose local http port the vhost proxies.
mode: DEPLOYMENT_MODE, ``compose`` or ``swarm``.
extra: further template variables, e.g. ``matrix_flavor``.
Returns:
The rendered nginx location block.
"""
template = (
"roles/web-app-matrix/templates/synapse.conf.j2"
if application_id == "web-app-matrix"
else "roles/web-app-prometheus/templates/nginx/locations.conf.j2"
)
http_port = get_app_conf(
applications=get_application_defaults(roles_dir=ROLES_DIR),
application_id=application_id,
config_path=f"services.{service_key}.ports.local.http",
strict=False,
default="",
)
domain = _template(
application_id, mode, {}, "{{ lookup('domain', application_id) }}"
)
return _render(
application_id,
template,
mode,
{"http_port": http_port, "domain": domain, **extra},
)
def _container_name(rendered: str) -> str:
found = _CONTAINER_NAME_SET.findall(rendered)
if len(found) != 1:
raise AssertionError(f"expected one 'set $container_name', found {found!r}")
return found[0]
def _lua_blocks(rendered: str) -> dict[str, list[str]]:
"""Collect the Lua body of every ``*_by_lua_block`` in rendered nginx config.
Args:
rendered: nginx configuration text.
Returns:
Directive name mapped to its bodies in document order.
"""
blocks: dict[str, list[str]] = {}
consumed = 0
for match in _LUA_BLOCK_OPEN.finditer(rendered):
if match.start() < consumed:
continue
depth, i, quote = 1, match.end(), None
while depth:
ch = rendered[i]
if quote:
if ch == "\\":
i += 2
continue
if ch == quote:
quote = None
elif rendered.startswith("--", i):
i = rendered.index("\n", i)
continue
elif ch in "\"'":
quote = ch
elif ch == "{":
depth += 1
elif ch == "}":
depth -= 1
i += 1
consumed = i
blocks.setdefault(match.group(1), []).append(rendered[match.end() : i - 1])
return blocks
def _lua_literal(value: object) -> str:
if value is None:
return "null"
if isinstance(value, str):
return json.dumps(value)
if isinstance(value, list):
return "{" + ", ".join(_lua_literal(v) for v in value) + "}"
if isinstance(value, Mapping):
return (
"{"
+ ", ".join(
f"[{json.dumps(k)}] = {_lua_literal(v)}" for k, v in value.items()
)
+ "}"
)
raise TypeError(f"no Lua literal for {type(value).__name__}")
def _harness_outcome(mode: str, keys: list[str]) -> dict:
"""Run the rendered Lua blocks under luajit against a canned Docker answer.
Args:
mode: DEPLOYMENT_MODE whose renders and ``_DOCKER_PS`` fixture are used.
keys: ``$container_name`` values to request ``/healthz/live`` and ``/metricz`` with.
Returns:
``enabled``, ``worker``, ``agent``, ``expired``, ``denied`` and ``stale_conf`` results
plus the ``dict``, ``live``, ``metricz`` and ``denied_metricz`` maps
keyed by container key, each value a tuple of the printed fields.
"""
conf = _lua_blocks(
_render(
PROMETHEUS_APP_ID,
"roles/web-app-prometheus/templates/nginx/prometheus.conf.j2",
mode,
{},
)
)
vhost = _lua_blocks(_vhost(PROMETHEUS_APP_ID, "prometheus", mode))
files = {
"init.lua": conf["init_by_lua_block"][0],
"init_worker.lua": conf["init_worker_by_lua_block"][0],
"live.lua": vhost["content_by_lua_block"][0],
"metricz.lua": vhost["content_by_lua_block"][-1],
"containers.json": json.dumps(_DOCKER_PS[mode]),
"containers.lua": "local null = ...\nreturn "
+ _lua_literal(_DOCKER_PS[mode])
+ "\n",
"harness.lua": _LUA_HARNESS,
}
with tempfile.TemporaryDirectory() as tmp:
work = Path(tmp)
for name, text in files.items():
(work / name).write_text(text)
result = subprocess.run(
["luajit", str(work / "harness.lua"), str(work), *keys],
capture_output=True,
text=True,
check=False,
)
if result.returncode != 0:
raise AssertionError(f"luajit failed in {mode}:\n{result.stderr}")
keyed = ("dict", "live", "metricz", "denied_metricz")
outcome: dict = {kind: {} for kind in keyed}
for fields in (line.split("\t") for line in result.stdout.splitlines()):
if fields[0] in keyed:
outcome[fields[0]][fields[1]] = tuple(fields[2:])
else:
outcome[fields[0]] = fields[1:]
return outcome
class TestPrometheusServicePresence(unittest.TestCase):
"""
All web-app-* and web-svc-* roles (except web-app-prometheus itself) must
@@ -642,6 +1096,145 @@ class TestDockerHealthCheck(unittest.TestCase):
)
class TestDockerHealthKey(unittest.TestCase):
CASES = (
("web-app-gitea", "gitea", {}, "gitea"),
(PROMETHEUS_APP_ID, "prometheus", {}, "prometheus"),
("web-app-matrix", "synapse", {"matrix_flavor": "compose"}, "synapse"),
("web-app-matrix", "synapse", {"matrix_flavor": "ansible"}, "matrix"),
)
def test_key_is_the_name_docker_reports_in_each_mode(self):
applications = get_application_defaults(roles_dir=ROLES_DIR)
for application_id, port_key, extra, docker_key in self.CASES:
expected = {
"compose": get_app_conf(
applications=applications,
application_id=application_id,
config_path=f"services.{docker_key}.name",
),
"swarm": f"{get_entity_name(application_id)}_{docker_key}",
}
for mode in _MODES:
with self.subTest(application_id=application_id, mode=mode, **extra):
self.assertEqual(
expected[mode],
_container_name(
_vhost(application_id, port_key, mode, **extra)
),
)
@unittest.skipUnless(shutil.which("luajit"), "luajit runs the rendered Lua blocks")
class TestDockerHealthLuaRuntime(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.keys = {
mode: {
"unhealthy": _container_name(_vhost("web-app-gitea", "gitea", mode)),
"healthy": _container_name(
_vhost(PROMETHEUS_APP_ID, "prometheus", mode)
),
"mixed": _container_name(
_vhost("web-app-matrix", "synapse", mode, matrix_flavor="compose")
),
"unknown": "not-on-this-node",
}
for mode in _MODES
}
cls.outcomes = {
mode: _harness_outcome(mode, list(cls.keys[mode].values()))
for mode in _MODES
}
def test_init_enables_the_privileged_agent(self):
for mode in _MODES:
with self.subTest(mode=mode):
self.assertEqual(["true"], self.outcomes[mode]["enabled"])
def test_request_workers_never_touch_the_docker_socket(self):
for mode in _MODES:
with self.subTest(mode=mode):
self.assertEqual(["0/0/0"], self.outcomes[mode]["worker"])
def test_privileged_agent_polls_the_docker_socket(self):
for mode in _MODES:
with self.subTest(mode=mode):
self.assertEqual(
[
_DOCKER_SOCKET,
"GET /v1.41/containers/json HTTP/1.0",
"0/15",
],
self.outcomes[mode]["agent"],
)
def test_poller_stores_state_under_the_key_the_vhost_reads(self):
for mode in _MODES:
keys = self.keys[mode]
with self.subTest(mode=mode):
self.assertEqual(
{
keys["unhealthy"]: ("down/60",),
keys["healthy"]: ("up/60",),
keys["mixed"]: ("up/60",),
},
{
key: self.outcomes[mode]["dict"].get(key)
for key in (keys["unhealthy"], keys["healthy"], keys["mixed"])
},
)
def test_live_and_stack_up_report_an_unhealthy_container_down(self):
for mode in _MODES:
keys = self.keys[mode]
outcome = self.outcomes[mode]
with self.subTest(mode=mode):
self.assertEqual(
{
keys["unhealthy"]: ("503", "0"),
keys["healthy"]: ("200", "1"),
keys["mixed"]: ("200", "1"),
keys["unknown"]: ("200", "1"),
},
{
key: (outcome["live"][key][0], outcome["metricz"][key][0])
for key in keys.values()
},
)
def test_entries_gauge_counts_the_unexpired_dict_entries(self):
for mode in _MODES:
outcome = self.outcomes[mode]
with self.subTest(mode=mode):
self.assertGreater(len(outcome["dict"]), 0)
self.assertEqual(
{str(len(outcome["dict"]))},
{fields[1] for fields in outcome["metricz"].values()},
)
def test_entries_gauge_drops_to_zero_once_the_poller_stops(self):
for mode in _MODES:
with self.subTest(mode=mode):
self.assertEqual(["0"], self.outcomes[mode]["expired"])
def test_entries_gauge_is_zero_when_the_socket_is_denied(self):
for mode in _MODES:
keys = self.keys[mode]
outcome = self.outcomes[mode]
with self.subTest(mode=mode):
self.assertEqual(["1", "1"], outcome["denied"])
self.assertEqual(
dict.fromkeys(keys.values(), ("200", "1", "0")),
outcome["denied_metricz"],
)
def test_metricz_still_collects_before_the_gauge_is_registered(self):
for mode in _MODES:
with self.subTest(mode=mode):
self.assertEqual(["true", "true"], self.outcomes[mode]["stale_conf"])
class TestNativeAppMetrics(unittest.TestCase):
"""
Applications that provide native Prometheus metrics MUST have a scrape job
@@ -0,0 +1,88 @@
"""Every worktree slot must pin a valid swarm lab subnet of its own.
`scripts/system/worktree/up.sh` derives `INFINITO_SWARM_LAB_SUBNET` from the
slot. A derivation that overflows an octet yields an address Docker rejects at
network creation, and one that lands in a reserved range breaks the swarm at
runtime: inside NETWORK_TOR_VIRTUAL_CIDR svc-net-tor redirects every TCP
connection to Tor's TransPort, so the nodes cannot reach each other.
"""
from __future__ import annotations
import ipaddress
import re
import unittest
from utils.cache.files import read_text
from utils.cache.yaml import load_yaml_any
from utils.env.parser import parse_static_env
from . import PROJECT_ROOT
_UP_SH = PROJECT_ROOT / "scripts/system/worktree/up.sh"
_NETWORKS = PROJECT_ROOT / "group_vars/all/08_networks.yml"
_MAX_SLOT_RE = re.compile(r"^max_slot=(?P<value>\d+)$", re.MULTILINE)
_LAB_PREFIX_RE = re.compile(r'^lab_prefix="(?P<template>[^"]+)"$', re.MULTILINE)
_LAB_SUBNET_RE = re.compile(
r"^INFINITO_SWARM_LAB_SUBNET=\$\{lab_prefix\}\.0/24$", re.MULTILINE
)
def _reserved_networks() -> dict[str, ipaddress.IPv4Network]:
networks = load_yaml_any(str(_NETWORKS), default_if_missing={})
reserved = {
"NETWORK_TOR_VIRTUAL_CIDR": networks["NETWORK_TOR_VIRTUAL_CIDR"],
"NETWORK_MESH_POOL": networks["NETWORK_MESH_POOL"],
}
for index, pool in enumerate(networks["NETWORK_DOCKER_ADDRESS_POOLS"]):
reserved[f"NETWORK_DOCKER_ADDRESS_POOLS[{index}]"] = pool["base"]
for index, pool in enumerate(networks["NETWORK_NESTED_DOCKER_ADDRESS_POOLS"]):
reserved[f"NETWORK_NESTED_DOCKER_ADDRESS_POOLS[{index}]"] = pool["base"]
reserved["NETWORK_NESTED_DOCKER_BIP"] = networks["NETWORK_NESTED_DOCKER_BIP"]
reserved["primary INFINITO_SWARM_LAB_SUBNET"] = parse_static_env(
PROJECT_ROOT / "default.env"
)["INFINITO_SWARM_LAB_SUBNET"]
reserved["worktree INFINITO_SUBNET range"] = "172.30.0.0/16"
return {
name: ipaddress.ip_network(value, strict=False)
for name, value in reserved.items()
}
class TestWorktreeSwarmLabSubnet(unittest.TestCase):
def setUp(self) -> None:
source = read_text(str(_UP_SH))
max_slot = _MAX_SLOT_RE.search(source)
lab_prefix = _LAB_PREFIX_RE.search(source)
self.assertIsNotNone(max_slot, f"{_UP_SH} declares no max_slot")
self.assertIsNotNone(lab_prefix, f"{_UP_SH} declares no lab_prefix")
self.assertRegex(source, _LAB_SUBNET_RE)
self.max_slot = int(max_slot.group("value"))
self.template = lab_prefix.group("template")
self.reserved = _reserved_networks()
def _subnet(self, slot: int) -> ipaddress.IPv4Network:
prefix = self.template.replace("${slot}", str(slot))
return ipaddress.ip_network(f"{prefix}.0/24")
def test_every_slot_pins_a_valid_distinct_unreserved_lab_subnet(self) -> None:
seen: dict[ipaddress.IPv4Network, int] = {}
for slot in range(1, self.max_slot + 1):
with self.subTest(slot=slot):
subnet = self._subnet(slot)
clashes = sorted(
name
for name, network in self.reserved.items()
if subnet.overlaps(network)
)
self.assertEqual(
clashes, [], f"slot {slot} lab {subnet} overlaps {clashes}"
)
self.assertNotIn(
subnet, seen, f"slot {slot} repeats slot {seen.get(subnet)}"
)
seen[subnet] = slot
if __name__ == "__main__":
unittest.main()
@@ -10,10 +10,13 @@ from unittest.mock import patch
from ansible.errors import AnsibleError
from plugins.lookup.applications import LookupModule as ApplicationsLookup
from plugins.lookup.email import LookupModule
from utils.cache import _reset_cache_for_tests
from utils.cache import applications as cache_applications
from utils.cache import base as cache_base
from utils.cache import users as cache_users
from utils.cache.carrier import APPLICATIONS_RENDERED_FACT
from utils.cache.yaml import dump_yaml_str
from utils.roles.mapping import ROLE_FILE_META_DOMAINS, ROLE_FILE_META_SERVICES
@@ -511,6 +514,88 @@ class TestProviderOnionIsClusterWide(unittest.TestCase):
self.assertTrue(result["tls"])
def _park(self, variables: dict) -> dict:
lookup = ApplicationsLookup()
lookup._templar = self.lookup._templar
return lookup.run(
[], variables=variables, roles_dir=str(self._tmp / "roles"), carrier=True
)[0]
def _provider_view(self, worker: dict, **overrides) -> dict:
view = dict(worker, inventory_hostname="mgr")
view["group_names"] = worker["hostvars"]["mgr"]["group_names"]
view.update(overrides)
return view
def _worker_with_carrier(self) -> dict:
worker = self._worker()
worker[APPLICATIONS_RENDERED_FACT] = self._park(worker)
return worker
def _run_counting_renders(self, worker: dict) -> tuple[dict, int]:
_reset_cache_for_tests()
with patch.object(
cache_applications,
"_render_with_templar",
wraps=cache_applications._render_with_templar,
) as render:
result = self._run(worker)
return result, render.call_count
def test_worker_reads_the_provider_view_from_the_provider_carrier(self) -> None:
self._seed(ports=self.STALWART_PORTS)
worker = self._worker_with_carrier()
expected, baseline_renders = self._run_counting_renders(worker)
worker["hostvars"]["mgr"][APPLICATIONS_RENDERED_FACT] = self._park(
self._provider_view(worker)
)
result, renders = self._run_counting_renders(worker)
self.assertEqual(baseline_renders, 1)
self.assertEqual(renders, 0)
self.assertEqual(result, expected)
def test_worker_renders_when_the_provider_carrier_key_differs(self) -> None:
self._seed(ports=self.STALWART_PORTS)
worker = self._worker_with_carrier()
expected, _ = self._run_counting_renders(worker)
worker["hostvars"]["mgr"][APPLICATIONS_RENDERED_FACT] = self._park(
self._provider_view(worker, group_names=["web-app-mailprov"])
)
result, renders = self._run_counting_renders(worker)
self.assertGreater(renders, 0)
self.assertEqual(result, expected)
self.assertFalse(result["tls"])
def test_worker_reads_the_provider_carrier_untemplated(self) -> None:
self._seed(ports=self.STALWART_PORTS)
worker = self._worker_with_carrier()
expected, _ = self._run_counting_renders(worker)
raw = {
"mgr": dict(
worker["hostvars"]["mgr"],
**{APPLICATIONS_RENDERED_FACT: self._park(self._provider_view(worker))},
)
}
worker["hostvars"] = _RawGetHostvars(worker["hostvars"], raw)
result, renders = self._run_counting_renders(worker)
self.assertEqual(renders, 0)
self.assertEqual(result, expected)
class _RawGetHostvars(dict):
def __init__(self, templated: dict, raw: dict) -> None:
super().__init__(templated)
self._raw = raw
def raw_get(self, host: str) -> dict:
return self._raw[host]
class _DummyTemplar:
def __init__(self, available_variables: dict[str, str]) -> None:
@@ -0,0 +1,38 @@
from __future__ import annotations
import re
import unittest
from utils.cache.files import read_text
from . import PROJECT_ROOT
ROLE = PROJECT_ROOT / "roles" / "svc-net-tor"
TEST_ENV = ROLE / "templates" / "test.env.j2"
TORRC = ROLE / "templates" / "torrc.j2"
TOR_DEFAULT_SOCKS_TIMEOUT = 120
class TestOnionPortWindow(unittest.TestCase):
def test_an_attempt_outlasts_tors_socks_timeout(self) -> None:
self.assertNotRegex(
read_text(str(TORRC)),
r"(?m)^SocksTimeout\b",
"torrc now sets SocksTimeout; ONION_PORT_TIMEOUT has to be derived "
"from it instead of from Tor's default",
)
declared = re.search(
r"(?m)^ONION_PORT_TIMEOUT=(\d+)$", read_text(str(TEST_ENV))
)
self.assertIsNotNone(declared, "test.env.j2 declares no ONION_PORT_TIMEOUT")
self.assertGreater(
int(declared.group(1)),
TOR_DEFAULT_SOCKS_TIMEOUT,
"a port probe attempt that ends before Tor's SocksTimeout walks away "
"from a rendezvous Tor is still building, and Tor never reaches its "
"own verdict or logs it",
)
if __name__ == "__main__":
unittest.main()
@@ -9,6 +9,7 @@ secrets, so neither may reach curl's command line.
from __future__ import annotations
import json
import re
import unittest
from typing import ClassVar
@@ -24,6 +25,14 @@ TASK = PROJECT_ROOT / "roles" / "web-app-stalwart" / "tasks" / "jmap_call.yml"
PASSWORD = 'p"a\\s$s`w#o[r]d é'
SWARM_ADDRESS = '"$(/usr/bin/resolve-container-id stalwart stalwart)"'
RESOLVED_ONCE = (
"STALWART_JMAP_URL",
"STALWART_ADMIN_USER",
"STALWART_ADMIN_PASSWORD",
)
def _tasks() -> list[dict]:
return load_yaml_any(str(TASK), default_if_missing=[])
@@ -37,6 +46,15 @@ def _call_task() -> dict:
return next(_module(t, "shell") for t in _tasks() if _module(t, "shell"))
def _client_task() -> dict:
return next(t for t in _tasks() if "stalwart_jmap_client" in _module(t, "set_fact"))
def _resolved_client(**variables) -> dict:
client = _module(_client_task(), "set_fact")["stalwart_jmap_client"]
return {key: _render(raw, **variables) for key, raw in client.items()}
def _assert_task() -> dict:
return next(t for t in _tasks() if _module(t, "assert"))
@@ -220,16 +238,68 @@ class TestJmapSecretsStayOffArgv(unittest.TestCase):
def test_stdin_config_round_trips_hostile_values(self):
body = json.dumps({"secret": 'q"\\x', "name": "Zoë"})
rendered = _render(
self.task["stdin"],
client = _resolved_client(
STALWART_JMAP_URL="http://localhost:8080/jmap/",
STALWART_ADMIN_USER="stalwartadmin",
STALWART_ADMIN_PASSWORD=PASSWORD,
stalwart_jmap_body=body,
)
rendered = _render(
self.task["stdin"], stalwart_jmap_client=client, stalwart_jmap_body=body
)
config = dict(line.split(" = ", 1) for line in rendered.splitlines())
self.assertEqual(json.loads(config["user"]), f"stalwartadmin:{PASSWORD}")
self.assertEqual(json.loads(config["data-binary"]), body)
class TestJmapClientResolvedOncePerRun(unittest.TestCase):
def setUp(self):
self.call = _call_task()
self.resolve = _client_task()
def test_the_call_templates_no_resolved_once_variable(self):
for field in ("cmd", "stdin"):
for name in RESOLVED_ONCE:
self.assertNotIn(name, self.call[field])
def test_the_client_carries_every_resolved_once_variable(self):
client = _module(self.resolve, "set_fact")["stalwart_jmap_client"]
for name in RESOLVED_ONCE:
self.assertTrue(any(name in raw for raw in client.values()), name)
def test_the_call_reads_only_keys_the_client_sets(self):
client = _module(self.resolve, "set_fact")["stalwart_jmap_client"]
used = set(
re.findall(
r"stalwart_jmap_client\.(\w+)", self.call["cmd"] + self.call["stdin"]
)
)
self.assertTrue(used)
self.assertLessEqual(used, set(client))
def test_the_client_resolves_only_while_undefined(self):
condition = "{{ (" + str(self.resolve.get("when", True)) + ") | bool }}"
self.assertTrue(_render(condition))
self.assertFalse(_render(condition, stalwart_jmap_client={"url": "u"}))
def test_the_client_resolution_is_masked(self):
self.assertEqual(
self.resolve["no_log"], "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
)
def test_a_swarm_address_reaches_the_shell_unresolved(self):
client = _resolved_client(
STALWART_JMAP_URL="http://localhost:8080/jmap/",
STALWART_ADMIN_USER="stalwartadmin",
STALWART_ADMIN_PASSWORD=PASSWORD,
)
cmd = _render(
self.call["cmd"],
STALWART_CONTAINER_ADDRESS=SWARM_ADDRESS,
stalwart_jmap_client=client,
)
self.assertTrue(cmd.startswith(f"container exec -i {SWARM_ADDRESS} curl "))
self.assertTrue(cmd.endswith(" http://localhost:8080/jmap/"))
if __name__ == "__main__":
unittest.main()