kevinveenbirkenbach/computer-playbook
1
0 Fork 0
mirror of https://github.com/kevinveenbirkenbach/computer-playbook.git synced 2026-11-06 02:54:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into update/web-app-suitecrm/8.10.2

Browse Source 
# Conflicts:
#	roles/web-app-suitecrm/meta/services.yml
This commit is contained in:
kevinveenbirkenbach committed 2026-10-08 19:40:07 +02:00
2 parents eeda40755e 66b3c9f50f
commit 1479011412
32 files changed
+661 -211
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

compose.yml
-1
View File
@@ -23,7 +23,6 @@ services:
build:
context: .
dockerfile: Dockerfile
network: host
args:
INFINITO_PARENT_IMAGE: ${INFINITO_PARENT_IMAGE:?Run 'make dotenv' to generate the .env single source of truth}
INFINITO_SRC_DIR: ${INFINITO_SRC_DIR:?Run 'make dotenv' to generate the .env single source of truth}
plugins/filter/native_metrics_target.py
-51
View File
@@ -1,51 +0,0 @@
from __future__ import annotations
from ansible.errors import AnsibleFilterError
from utils.roles.entity.name import get_entity_name
def native_metrics_target(app_id: str, applications: dict) -> str:
"""Return the container-to-container Prometheus scrape target for a native-metrics app.
Prometheus connects to the app's Docker network (declared in prometheus's
compose.yml.j2 via the native_metrics_apps loop) and scrapes by container
name and internal port — no host port binding required, no nginx/OAuth2 in
the path.
Usage in a per-app prometheus.yml.j2 fragment:
targets: ["{{ native_prometheus_application_id | native_metrics_target(lookup('applications')) }}"]
When the service key differs from the app's entity name (e.g. matrix
uses service key "synapse"), set services.prometheus.native_metrics.service_key
in the app's meta/services.yml to override.
"""
app_conf = applications.get(app_id, {})
services = app_conf.get("services", {}) if isinstance(app_conf, dict) else {}
native_metrics_conf = (
services.get("prometheus", {}) if isinstance(services, dict) else {}
).get("native_metrics", {})
service_key = native_metrics_conf.get("service_key") or get_entity_name(app_id)
container = (
services.get(service_key, {}).get("name")
if isinstance(services, dict)
else None
)
if not container:
raise AnsibleFilterError(
f"native_metrics_target: no services.{service_key}.name for '{app_id}'"
)
port = native_metrics_conf.get("port")
if port is None:
raise AnsibleFilterError(
f"native_metrics_target: no services.prometheus.native_metrics.port for '{app_id}'"
)
return f"{container}:{port}"
class FilterModule:
def filters(self):
return {"native_metrics_target": native_metrics_target}
plugins/lookup/native_metrics_apps.py
+30 -2
View File
@@ -1,12 +1,35 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from ansible.plugins.loader import lookup_loader
from ansible.plugins.lookup import LookupBase
from utils.cache import ROLES_DIR
from utils.cache.yaml import load_yaml_any
from utils.roles.applications.config import get as get_app_conf
from utils.roles.mapping import ROLE_FILE_VARS_MAIN
def compose_mode_force(app_id: str) -> str:
"""Return the role's own deploy-mode override.
Args:
app_id: the role directory name.
Returns:
The literal ``compose_mode_force`` from the role's vars, or ``""`` when
it is absent or a Jinja expression, which cannot be resolved from the
scraping role's variable scope and so leaves the override invisible here.
"""
path = ROLES_DIR / app_id / ROLE_FILE_VARS_MAIN
if not path.is_file():
return ""
data = load_yaml_any(str(path), default_if_missing={})
raw = data.get("compose_mode_force") if isinstance(data, Mapping) else None
value = "" if raw is None else str(raw).strip()
return "" if "{{" in value else value
class LookupModule(LookupBase):
@@ -16,8 +39,10 @@ class LookupModule(LookupBase):
2. a prometheus.yml.j2 template at roles/<app_id>/templates/
3. reachable from this prometheus: NOT on a node-local force_bridge
network under swarm, which a swarm prometheus can neither join nor
resolve. This is the single point that gates force_bridge apps out of
the native-metrics precreate + scrape.
resolve, and NOT pinned to compose by its own compose_mode_force while
the cluster runs swarm, which would make the scrape target name the
wrong object. This is the single point that gates unreachable apps out
of the native-metrics precreate + scrape.
Used by web-app-prometheus/templates/configuration/prometheus.yml.j2 to auto-discover apps
that expose a native /metrics endpoint without hardcoding each app name.
@@ -78,6 +103,9 @@ class LookupModule(LookupBase):
if is_swarm and bool(force_bridge):
continue
if is_swarm and compose_mode_force(app_id) == "compose":
continue
result.append(app_id)
return [result]
plugins/lookup/native_metrics_target.py
+72
View File
@@ -0,0 +1,72 @@
"""Lookup ``native_metrics_target``: scrape target for an app that exposes its
own metrics port. Reads ``services.prometheus.native_metrics`` for the service
key and port, then delegates host resolution to ``scrape_target`` so swarm and
compose stay on one implementation.
Usage in a per-app prometheus.yml.j2 fragment:
targets: ["{{ lookup('native_metrics_target', native_prometheus_application_id) }}"]
"""
from __future__ import annotations
from typing import Any
from ansible.errors import AnsibleError
from ansible.plugins.loader import lookup_loader
from ansible.plugins.lookup import LookupBase
from utils.roles.applications.config import get
from utils.roles.entity.name import get_entity_name
class LookupModule(LookupBase):
def run(
self,
terms: list[Any] | None,
variables: dict[str, Any] | None = None,
**kwargs: Any,
) -> list[str]:
if not terms:
raise AnsibleError(
"native_metrics_target lookup requires the application_id term"
)
application_id = str(terms[0]).strip()
if not application_id:
raise AnsibleError(
"native_metrics_target: application_id must be non-empty"
)
vars_ = variables or getattr(self._templar, "available_variables", {}) or {}
applications = lookup_loader.get(
"applications", loader=self._loader, templar=getattr(self, "_templar", None)
).run([], variables=vars_)[0]
def conf(key: str) -> str:
value = get(
applications=applications,
application_id=application_id,
config_path=f"services.prometheus.native_metrics.{key}",
strict=False,
default="",
)
return "" if value is None else str(value).strip()
port = conf("port")
if not port:
raise AnsibleError(
"native_metrics_target: no "
f"services.prometheus.native_metrics.port for {application_id!r}"
)
service_key = conf("service_key") or get_entity_name(application_id)
if not service_key:
raise AnsibleError(
f"native_metrics_target: cannot derive service key for {application_id!r}"
)
return lookup_loader.get(
"scrape_target",
loader=self._loader,
templar=getattr(self, "_templar", None),
).run([application_id, service_key], variables=vars_, port=port)
plugins/lookup/scrape_target.py
+13 -4
View File
@@ -1,6 +1,7 @@
"""Lookup ``scrape_target``: mode-aware ``<host>:<port>`` for a Prometheus
scrape target. Swarm resolves ``tasks.<entity>_<service_key>`` (per-task DNS);
compose uses ``compose_host`` or the service container name."""
compose uses ``compose_host`` or the service container name. ``port`` overrides
the ``services.<service_key>.ports.internal.<port_kind>`` lookup."""
from __future__ import annotations
@@ -37,12 +38,20 @@ class LookupModule(LookupBase):
if not service_key:
raise AnsibleError("scrape_target: service_key must be non-empty")
port_kind = (
explicit_port_kind = (
_as_str(terms[2]) if len(terms) > 2 else _as_str(kwargs.get("port_kind"))
) or "http"
)
port_kind = explicit_port_kind or "http"
compose_host = (
_as_str(terms[3]) if len(terms) > 3 else _as_str(kwargs.get("compose_host"))
)
port_override = _as_str(kwargs.get("port"))
if port_override and explicit_port_kind:
raise AnsibleError(
f"scrape_target: {application_id!r} passed both port={port_override!r} "
f"and port_kind={explicit_port_kind!r}; port wins and port_kind would be "
"silently dropped, so pass only one"
)
vars_ = variables or getattr(self._templar, "available_variables", {}) or {}
templar = getattr(self, "_templar", None)
@@ -57,7 +66,7 @@ class LookupModule(LookupBase):
"applications", loader=self._loader, templar=getattr(self, "_templar", None)
).run([], variables=vars_)[0]
port = _as_str(
port = port_override or _as_str(
get(
applications=applications,
application_id=application_id,
roles/test-e2e-playwright/meta/services.yml
+1 -1
View File
@@ -4,4 +4,4 @@ test-e2e-playwright:
lifecycle: beta
playwright:
image: mcr.microsoft.com/playwright
version: "v1.63.0-noble"
version: "v1.64.0-noble"
roles/web-app-gitea/templates/prometheus.yml.j2
+1 -1
View File
@@ -9,6 +9,6 @@
bearer_token: "{{ _token }}"
{% endif %}
static_configs:
- targets: ["{{ native_prometheus_application_id | native_metrics_target(lookup('applications')) }}"]
- targets: ["{{ lookup('native_metrics_target', native_prometheus_application_id) }}"]
labels:
app: "{{ native_prometheus_application_id }}"
roles/web-app-matrix/meta/services.yml
+2 -2
View File
@@ -104,8 +104,8 @@ prometheus:
shared: "{{ 'web-app-prometheus' in group_names }}"
native_metrics:
enabled: true
port: 9000
service_key: synapse
port: "{{ 9000 if lookup('config', 'web-app-matrix', 'services.matrix.flavor') == 'compose' else 9100 }}"
service_key: "{{ 'synapse' if lookup('config', 'web-app-matrix', 'services.matrix.flavor') == 'compose' else 'matrix' }}"
communication:
channel: true
# nocheck: playwright-service-flag: shared provider entity; the app itself is covered by this role's own Playwright spec
roles/web-app-matrix/templates/flavor/ansible/vars.yml.j2
+4
View File
@@ -153,5 +153,9 @@ matrix_bridge_mautrix_twitter_enabled: {{ MATRIX_BRIDGE.mautrix_twitter | string
matrix_bridge_mautrix_whatsapp_enabled: {{ MATRIX_BRIDGE.mautrix_whatsapp | string | lower }}
matrix_bridge_mautrix_gitter_enabled: {{ MATRIX_BRIDGE.gitter | string | lower }}
matrix_synapse_metrics_enabled: true
matrix_synapse_container_metrics_api_host_bind_port: "0.0.0.0:{{ MATRIX_SYNAPSE_METRICS_PORT }}"
matrix_synapse_metrics_port: {{ MATRIX_SYNAPSE_METRICS_PORT }}
jitsi_enabled: {{ MATRIX_CONFERENCING_JITSI | string | lower }}
matrix_rtc_enabled: {{ MATRIX_CONFERENCING_ELEMENT_CALL | string | lower }}
roles/web-app-matrix/templates/prometheus.yml.j2
+1 -3
View File
@@ -1,12 +1,10 @@
# Matrix Synapse — server and federation metrics.
# Requires enable_metrics: true in homeserver.yaml.
# Enable via native_metrics.enabled: true in inventory.
# Scraped container-to-container: prometheus joins the matrix Docker network and
# reaches synapse directly at container_name:port — no host binding, no nginx/OAuth2.
- job_name: "matrix-synapse"
scheme: http
metrics_path: /_synapse/metrics
static_configs:
- targets: ["{{ native_prometheus_application_id | native_metrics_target(lookup('applications')) }}"]
- targets: ["{{ lookup('native_metrics_target', native_prometheus_application_id) }}"]
labels:
app: "{{ native_prometheus_application_id }}"
roles/web-app-matrix/vars/main.yml
+1
View File
@@ -82,6 +82,7 @@ MATRIX_BRIDGE: "{{ lookup('config', application_id, 'services.matrix.plugins') }
MATRIX_CONFERENCING_JITSI: "{{ lookup('config', application_id, 'services.matrix.conferencing.jitsi') | bool }}"
MATRIX_CONFERENCING_ELEMENT_CALL: "{{ lookup('config', application_id, 'services.matrix.conferencing.element_call') | bool }}"
MATRIX_SYNAPSE_METRICS_PORT: "{{ lookup('config', application_id, 'services.prometheus.native_metrics.port') }}"
MATRIX_LDAP_ENABLED: "{{ lookup('config', application_id, 'services.ldap.enabled') | bool }}"
roles/web-app-mattermost/templates/prometheus.yml.j2
+1 -3
View File
@@ -1,11 +1,9 @@
# Mattermost — application and Go runtime metrics.
# Scraped container-to-container: prometheus joins the mattermost Docker network and
# reaches the app directly at container_name:port — no host binding, no nginx/OAuth2.
# Requires MM_METRICSSETTINGS_ENABLE=true and MM_METRICSSETTINGS_LISTENADDRESS (set in env.j2 when enabled).
- job_name: "mattermost"
scheme: http
metrics_path: /metrics
static_configs:
- targets: ["{{ native_prometheus_application_id | native_metrics_target(lookup('applications')) }}"]
- targets: ["{{ lookup('native_metrics_target', native_prometheus_application_id) }}"]
labels:
app: "{{ native_prometheus_application_id }}"
roles/web-app-nextcloud/meta/variants.yml
+2 -2
View File
@@ -1097,7 +1097,7 @@
enabled: false
shared: false
addons: *addons_static_off
# Variant 11: bigbluebutton
# Variant 11: bigbluebutton - clearnet, because bigbluebutton rides no node onion
- services:
container_backup:
enabled: false
@@ -1183,7 +1183,7 @@
enabled: false
shared: false # nocheck: variants-static-override backup shared stays literally true
tor:
enabled: true
enabled: false
hermes:
enabled: false
shared: false
roles/web-app-openbao/templates/prometheus.yml.j2
+1 -1
View File
@@ -4,6 +4,6 @@
params:
format: ["prometheus"]
static_configs:
- targets: ["{{ native_prometheus_application_id | native_metrics_target(lookup('applications')) }}"]
- targets: ["{{ lookup('native_metrics_target', native_prometheus_application_id) }}"]
labels:
app: "{{ native_prometheus_application_id }}"
roles/web-app-peertube/templates/proxy.conf.j2
+2
View File
@@ -51,4 +51,6 @@ server {
try_files /dev/null {{ webserver_websocket_location }};
}
{% include 'roles/web-app-prometheus/templates/nginx/locations.conf.j2' %}
}
roles/web-app-pixelfed/meta/services.yml
+1 -1
View File
@@ -49,7 +49,7 @@ pixelfed:
shared: true
enabled: true # nocheck: dynamic-flag
image: zknt/pixelfed
version: "20261007"
version: "20261008"
name: pixelfed
min_storage: 588MB
backup:
roles/web-app-prometheus/files/playwright/playwright.spec.js
+22 -2
View File
@@ -211,7 +211,7 @@ test("prometheus: biber is denied access after sso login", async ({ browser }) =
// -----------------------------------------------------------------------------
const prometheusTargetRoles = (() => {
const raw = process.env.PROMETHEUS_TARGET_ROLES_JSON || "[]";
const raw = decodeDotenvQuotedValue(process.env.PROMETHEUS_TARGET_ROLES_JSON || "[]");
try {
const parsed = JSON.parse(raw);
return Array.isArray(parsed) ? parsed : [];
@@ -220,7 +220,7 @@ const prometheusTargetRoles = (() => {
}
})();
test("prometheus scrape: every consumer role reports up=1", async ({ page }) => {
test("prometheus scrape: every consumer role reports up=1 and passes its probe", async ({ page }) => {
test.skip(prometheusTargetRoles.length === 0, "no prometheus consumer roles in inventory");
const expectedPrometheusBaseUrl = prometheusBaseUrl.replace(/\/$/, "");
@@ -279,6 +279,26 @@ test("prometheus scrape: every consumer role reports up=1", async ({ page }) =>
failures,
`prometheus scrape failures:\n - ${failures.join("\n - ")}`
).toEqual([]);
const probeUrl = `${expectedPrometheusBaseUrl}/api/v1/query?query=${encodeURIComponent('probe_success{job="blackbox-healthz"}')}`;
await expect
.poll(
async () => {
const response = await page.request.get(probeUrl, { ignoreHTTPSErrors: true });
const probes = response.ok() ? (await response.json())?.data?.result || [] : [];
return prometheusTargetRoles
.filter((target) => {
const own = probes.filter((entry) => entry?.metric?.app === target.id);
return own.length === 0 || own.some((entry) => entry?.value?.[1] !== "1");
})
.map((target) => target.id);
},
{
timeout: resolveTimeout(120_000),
message: "roles whose /healthz/ready probe is missing or failing (probe_success != 1)",
},
)
.toEqual([]);
});
// Persona scenarios.
roles/web-app-prometheus/templates/configuration/blackbox.yml.j2
+2 -2
View File
@@ -24,7 +24,7 @@ modules:
method: GET
follow_redirects: false
fail_if_ssl: false
fail_if_not_ssl: {{ lookup('tls', application_id, 'enabled') | bool | lower }}
fail_if_not_ssl: false
tls_config:
insecure_skip_verify: {{ not (lookup('tls', application_id, 'enabled') | bool) }}
insecure_skip_verify: {{ ((TLS_MODE | lower) == 'self_signed' and not lookup('ca_injected', application_id)) | lower }}
preferred_ip_protocol: "ip4"
roles/web-app-prometheus/templates/configuration/prometheus.yml.j2
+4 -3
View File
@@ -41,20 +41,21 @@ scrape_configs:
{% endfor %}
{% set canonical_map = lookup('applications') | canonical_domains_map(DOMAIN_PRIMARY) %}
{% set canonical_map = lookup('domains') %}
{% set probed_roles = lookup('roles_with_service', 'prometheus') | map(attribute='id') | list %}
- job_name: "blackbox-healthz"
metrics_path: /probe
params:
module: [http_ready]
static_configs:
{% for _application_id in CURRENT_PLAY_APPLICATIONS | list | sort %}
{% if _application_id != application_id %}
{% if _application_id in probed_roles %}
{% set domains_raw = canonical_map.get(_application_id, []) %}
{% set canonical_probe_domains = (domains_raw.values() | list) if (domains_raw is mapping) else domains_raw %}
{% if canonical_probe_domains | length > 0 %}
- targets:
{% for domain in canonical_probe_domains %}
- "{{ lookup('tls', application_id, 'protocols.web') }}://{{ domain }}/healthz/ready"
- "{{ lookup('tls', domain, 'url.base') }}healthz/ready"
{% endfor %}
labels:
app: "{{ _application_id }}"
roles/web-app-seaweedfs/meta/services.yml
+12 -6
View File
@@ -27,18 +27,24 @@ sso:
ldap:
enabled: false
shared: false
# nocheck: variants-coverage — the onion gate is a hard invariant; no variant may force this consumer on a tor node.
# nocheck: bond-baseline — same reason: variant 0 must not pin this consumer on.
logout:
bond: 1
enabled: "{{ 'web-svc-logout' in group_names }}"
shared: "{{ 'web-svc-logout' in group_names }}"
enabled: "{{ ('web-svc-logout' in group_names) and ('svc-net-tor' not in group_names) }}"
shared: "{{ ('web-svc-logout' in group_names) and ('svc-net-tor' not in group_names) }}"
# nocheck: variants-coverage — the onion gate is a hard invariant; no variant may force this consumer on a tor node.
# nocheck: bond-baseline — same reason: variant 0 must not pin this consumer on.
dashboard:
bond: 1
enabled: "{{ 'web-app-dashboard' in group_names }}"
shared: "{{ 'web-app-dashboard' in group_names }}"
enabled: "{{ ('web-app-dashboard' in group_names) and ('svc-net-tor' not in group_names) }}"
shared: "{{ ('web-app-dashboard' in group_names) and ('svc-net-tor' not in group_names) }}"
# nocheck: variants-coverage — the onion gate is a hard invariant; no variant may force this consumer on a tor node.
# nocheck: bond-baseline — same reason: variant 0 must not pin this consumer on.
prometheus:
bond: 1
enabled: "{{ 'web-app-prometheus' in group_names }}"
shared: "{{ 'web-app-prometheus' in group_names }}"
enabled: "{{ ('web-app-prometheus' in group_names) and ('svc-net-tor' not in group_names) }}"
shared: "{{ ('web-app-prometheus' in group_names) and ('svc-net-tor' not in group_names) }}"
# nocheck: playwright-service-flag — central object-store provider entity
seaweedfs:
healthcheck:
roles/web-app-seaweedfs/meta/variants.yml
+6 -6
View File
@@ -9,14 +9,14 @@
enabled: true
shared: true
logout:
enabled: true
shared: true
enabled: "{{ 'svc-net-tor' not in group_names }}"
shared: "{{ 'svc-net-tor' not in group_names }}"
dashboard:
enabled: true
shared: true
enabled: "{{ 'svc-net-tor' not in group_names }}"
shared: "{{ 'svc-net-tor' not in group_names }}"
prometheus:
enabled: true
shared: true
enabled: "{{ 'svc-net-tor' not in group_names }}"
shared: "{{ 'svc-net-tor' not in group_names }}"
- services:
container_backup:
enabled: false
roles/web-app-socialhome/meta/services.yml
+1 -1
View File
@@ -13,7 +13,7 @@ socialhome:
# nocheck: default-placement-manager embedded sqlite in a node-local volume; no clustered storage
placement: manager
image: ghcr.io/social-home-io/socialhome
version: "2026.10.7"
version: "2026.10.8"
ports:
local:
http: 8089
roles/web-svc-coturn/meta/services.yml
+8 -6
View File
@@ -36,21 +36,23 @@ gateway:
mem_reservation: 256m
mem_limit: 512m
pids_limit: 256
# nocheck: dynamic-flag
# nocheck: dynamic-flag coturn runs with user_auth_enabled false, so it keeps no credential store to back; the entry stays literal until auth is switched on and no template references redis today.
redis:
backup:
disabled: true
enabled: false
shared: "{{ 'svc-db-redis' in group_names }}"
# nocheck: variants-coverage — the tor conjunct is the clearnet-variant-tor invariant below, so no variant may pin the true polarity on an onion node
# nocheck: bond-baseline — same reason: variant 0 must not pin the scrape consumer on
prometheus:
bond: 1
enabled: "{{ 'web-app-prometheus' in group_names }}"
shared: "{{ 'web-app-prometheus' in group_names }}"
enabled: "{{ ('web-app-prometheus' in group_names) and ('svc-net-tor' not in group_names) }}"
shared: "{{ ('web-app-prometheus' in group_names) and ('svc-net-tor' not in group_names) }}"
# nocheck: clearnet-variant-tor — STUN/TURN on UDP plus a 20000-port relay range, which no onion descriptor can publish; the only HTTP is a gateway returning a plaintext OK. No dashboard, logout, css or matomo block and no Playwright spec, so an onion round claims no visitable surface for it. Its one consumer declaration, prometheus, carries the tor gate.
tor:
enabled: false # nocheck: dynamic-flag
shared: false # nocheck: dynamic-flag
# nocheck: dynamic-flag
enabled: false # nocheck: dynamic-flag no group can turn this on: see the clearnet-variant-tor reason above
shared: false # nocheck: dynamic-flag same
# nocheck: playwright-service-flag, dynamic-flag infrastructure backup consumer
container_backup: # nocheck: bond-baseline backup host joins outside the variant topology
bond: 1
roles/web-svc-coturn/meta/variants.yml
+2 -2
View File
@@ -9,8 +9,8 @@
container_backup:
enabled: true
prometheus:
enabled: true
shared: true
enabled: "{{ 'svc-net-tor' not in group_names }}"
shared: "{{ 'svc-net-tor' not in group_names }}"
redis:
shared: true
tor:
roles/web-svc-mirror/meta/services.yml
+2
View File
@@ -22,6 +22,8 @@ prometheus:
bond: 1
enabled: "{{ 'web-app-prometheus' in group_names }}"
shared: "{{ 'web-app-prometheus' in group_names }}"
# nocheck: prometheus-scrape
scrape: false
# nocheck: playwright-service-flag
# nocheck: variants-coverage — sso-variant-tor requires the onion on in every variant, so the false polarity has no variant left to live in
tor:
tests/lint/ansible/roles/meta/test_clearnet_role_forbids_variant_tor.py
+145
View File
@@ -0,0 +1,145 @@
"""Lint guard: a variant binding a clearnet-only role must not pin the onion on.
A role whose ``meta/services.yml`` pins ``tor.enabled`` to a literal false rides
no node onion, so on a tor deployment its surface lives at a clearnet host that
no onion browser reaches, and the dashboard's own
``frame-src 'self' http://*.<onion>`` forbids embedding it. A variant that
enables such a role and pins ``services.tor.enabled: true`` therefore declares a
combination the deployment cannot hold: the round advertises a tile, a tracker,
a stylesheet and a scrape target for a surface that is not there.
``test_sso_variant_requires_tor`` states the complementary half - every variant
of a role that has an onion MUST pin it on - and exempts the clearnet-only role
itself. This rule covers the consumers it does not: the variants that bind one.
The project already applies the rule by hand. ``web-app-nextcloud``'s variants
head carries "variant 8 pairs PeerTube, which is clearnet-only since 8ffb1f567,
so that round cannot pin the onion on".
Suppression (see ``docs/contributing/actions/testing/suppression.md``):
* ``# nocheck: clearnet-variant-tor`` in the CLEARNET role's
``meta/services.yml``, which exempts it everywhere: a relay with no browser
surface of its own carries no tile and no injected asset, so an onion round
may deploy it without claiming anything about it.
* the same marker in a consumer's ``meta/variants.yml`` for a case that is
sound only there.
The marker belongs on the bound role rather than the consumer whenever the
reason is the role's own shape, because a consumer-side marker would also
silence the pairings of every other clearnet role in the same file.
"""
from __future__ import annotations
import unittest
from typing import TYPE_CHECKING, Any
import yaml
from utils.annotations.suppress import line_has_rule
from utils.cache.files import read_text
from utils.cache.yaml import load_yaml_str
from utils.roles.entity.name import get_entity_name
from utils.roles.mapping import ROLE_FILE_META_SERVICES, ROLE_FILE_META_VARIANTS
from . import PROJECT_ROOT
if TYPE_CHECKING:
from pathlib import Path
ROLES_DIR = PROJECT_ROOT / "roles"
_RULE = "clearnet-variant-tor"
def _load_yaml(path: Path) -> Any:
if not path.is_file():
return None
try:
text = read_text(str(path))
except UnicodeDecodeError:
return None
if not text.strip():
return None
try:
return load_yaml_str(text)
except yaml.YAMLError:
return None
def _suppressed(paths: tuple[Path, ...]) -> bool:
for path in paths:
if not path.is_file():
continue
if any(
line_has_rule(line, _RULE) for line in read_text(str(path)).splitlines()
):
return True
return False
def _flag(variant: Any, service: str) -> Any:
services = variant.get("services") if isinstance(variant, dict) else None
entry = services.get(service) if isinstance(services, dict) else None
return entry.get("enabled") if isinstance(entry, dict) else None
def clearnet_only_keys() -> dict[str, str]:
"""Service key to role id, for every clearnet-only role the rule covers.
Returns:
The key a consumer's variant would bind it under, mapped to the role
that declares itself clearnet-only. A role carrying the marker in its
own ``meta/services.yml`` is left out, so it is exempt everywhere.
"""
found: dict[str, str] = {}
for role_dir in sorted(p for p in ROLES_DIR.iterdir() if p.is_dir()):
services_path = role_dir / ROLE_FILE_META_SERVICES
services = _load_yaml(services_path)
tor = services.get("tor") if isinstance(services, dict) else None
if not isinstance(tor, dict) or tor.get("enabled") is not False:
continue
if _suppressed((services_path,)):
continue
key = get_entity_name(role_dir.name)
if key:
found[key] = role_dir.name
return found
class TestClearnetRoleForbidsVariantTor(unittest.TestCase):
def test_no_variant_pins_the_onion_while_binding_a_clearnet_role(self) -> None:
clearnet = clearnet_only_keys()
offenders: list[str] = []
for role_dir in sorted(p for p in ROLES_DIR.iterdir() if p.is_dir()):
variants_path = role_dir / ROLE_FILE_META_VARIANTS
variants = _load_yaml(variants_path)
if not isinstance(variants, list) or not variants:
continue
if _suppressed((variants_path,)):
continue
for index, variant in enumerate(variants):
if _flag(variant, "tor") is not True:
continue
for key, bound_role in sorted(clearnet.items()):
if _flag(variant, key) is True:
offenders.append(
f"{role_dir.name} variant {index}: pins "
f"services.tor.enabled true while enabling {key!r} "
f"({bound_role}), which pins tor.enabled false"
)
self.assertEqual(
offenders,
[],
"A variant may not pin the onion on while binding a role that rides "
"none; that round advertises a surface no onion browser can reach. "
f"Turn tor off for the variant, or suppress with '# nocheck: {_RULE}' "
"naming why the pairing is sound:\n " + "\n ".join(offenders),
)
if __name__ == "__main__":
unittest.main()
tests/lint/repository/test_compose_build_needs_no_host_network.py
+60
View File
@@ -0,0 +1,60 @@
"""Compose hands a build to ``docker buildx bake`` and grants it only the
``fs.read`` entitlement of the build context. Since Buildx 0.38.0 a block with
``network: host`` stops there with ``additional privileges requested``, and
declaring ``build.entitlements`` or setting ``COMPOSE_BAKE=false`` does not
change that, so such a service cannot be built through ``make compose-up``.
"""
from __future__ import annotations
import unittest
from pathlib import Path
from utils.cache.files import iter_project_files
from utils.cache.yaml import load_yaml_any
from . import PROJECT_ROOT
STACK_FILE = str(PROJECT_ROOT / "compose.yml")
OVERRIDES = f"{PROJECT_ROOT / 'compose'}/"
STACK_BUILD = "compose.yml: infinito"
def _builds(path):
"""Yield ``(label, build)`` for every service of a compose file that builds.
Args:
path: compose file to read; a YAML file without ``services`` yields nothing.
"""
document = load_yaml_any(path)
if not isinstance(document, dict):
return
for name, service in (document.get("services") or {}).items():
build = service.get("build")
if isinstance(build, dict):
yield f"{Path(path).relative_to(PROJECT_ROOT)}: {name}", build
class TestComposeBuildNeedsNoHostNetwork(unittest.TestCase):
def test_no_build_block_asks_for_host_networking(self) -> None:
compose_files = [
path
for path in sorted(iter_project_files(extensions=(".yml",)))
if path == STACK_FILE or path.startswith(OVERRIDES)
]
builds = dict(hit for path in compose_files for hit in _builds(path))
self.assertIn(STACK_BUILD, builds, "the stack build block was not scanned")
offenders = [
label for label, build in builds.items() if build.get("network") == "host"
]
self.assertEqual(
[],
offenders,
"Compose cannot grant bake the network.host entitlement "
"(Buildx >= 0.38.0 refuses the build); drop 'network: host' from: "
f"{offenders}",
)
if __name__ == "__main__":
unittest.main()
tests/unit/python/plugins/filter/test_native_metrics_target.py
-111
View File
@@ -1,111 +0,0 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from ansible.errors import AnsibleFilterError
from plugins.filter.native_metrics_target import native_metrics_target
def _make_applications(
app_id: str,
container: str,
port: int,
entity_name: str,
service_key: str | None = None,
) -> dict:
native_metrics: dict = {"port": port}
if service_key is not None:
native_metrics["service_key"] = service_key
return {
app_id: {
"services": {
entity_name: {"name": container},
"prometheus": {"native_metrics": native_metrics},
}
}
}
class TestNativeMetricsTargetSuccess(unittest.TestCase):
def test_returns_container_colon_port(self):
apps = _make_applications("web-app-gitea", "gitea", 3000, "gitea")
with patch(
"plugins.filter.native_metrics_target.get_entity_name", return_value="gitea"
):
result = native_metrics_target("web-app-gitea", apps)
self.assertEqual(result, "gitea:3000")
def test_different_app(self):
apps = _make_applications(
"web-app-mattermost", "mattermost", 8067, "mattermost"
)
with patch(
"plugins.filter.native_metrics_target.get_entity_name",
return_value="mattermost",
):
result = native_metrics_target("web-app-mattermost", apps)
self.assertEqual(result, "mattermost:8067")
def test_service_key_override(self):
apps = _make_applications(
"web-app-matrix", "matrix-synapse", 9000, "synapse", service_key="synapse"
)
with patch(
"plugins.filter.native_metrics_target.get_entity_name",
return_value="matrix",
):
result = native_metrics_target("web-app-matrix", apps)
self.assertEqual(result, "matrix-synapse:9000")
class TestNativeMetricsTargetErrors(unittest.TestCase):
def test_raises_when_container_name_missing(self):
apps = {
"web-app-gitea": {
"services": {
"prometheus": {"native_metrics": {"port": 3000}},
}
}
}
with (
patch(
"plugins.filter.native_metrics_target.get_entity_name",
return_value="gitea",
),
self.assertRaises(AnsibleFilterError),
):
native_metrics_target("web-app-gitea", apps)
def test_raises_when_port_missing(self):
apps = {
"web-app-gitea": {
"services": {
"gitea": {"name": "gitea"},
"prometheus": {"native_metrics": {}},
}
}
}
with (
patch(
"plugins.filter.native_metrics_target.get_entity_name",
return_value="gitea",
),
self.assertRaises(AnsibleFilterError),
):
native_metrics_target("web-app-gitea", apps)
def test_raises_when_app_not_in_applications(self):
with (
patch(
"plugins.filter.native_metrics_target.get_entity_name",
return_value="gitea",
),
self.assertRaises(AnsibleFilterError),
):
native_metrics_target("web-app-gitea", {})
if __name__ == "__main__":
unittest.main()
tests/unit/python/plugins/lookup/test_native_metrics_target.py
+119
View File
@@ -0,0 +1,119 @@
"""Tests for the native_metrics_target lookup.
The lookup delegates host resolution to scrape_target, so these run the real
scrape_target rather than a stub: the regression being guarded is that swarm
must not receive a compose container name, and only the delegation proves it.
"""
from __future__ import annotations
import unittest
from typing import Any
from unittest.mock import MagicMock, patch
from ansible.errors import AnsibleError
from plugins.lookup.native_metrics_target import LookupModule
from plugins.lookup.scrape_target import LookupModule as ScrapeTargetLookup
ENTITIES = {"web-app-matrix": "matrix", "web-app-gitea": "gitea"}
class _Templar:
def __init__(self, vars_=None):
self.available_variables = vars_ or {}
def template(self, value, **_):
return value
def _apps(native_metrics: dict[str, Any]) -> dict[str, Any]:
return {
"web-app-matrix": {
"services": {
"prometheus": {"native_metrics": native_metrics},
"synapse": {
"name": "matrix-synapse",
"ports": {"internal": {"http": 8008}},
},
},
},
}
def _run(terms, *, mode, apps):
variables = {"DEPLOYMENT_MODE": mode}
lookup = LookupModule()
lookup._templar = _Templar(variables)
lookup._loader = MagicMock()
def _dispatch(name, **_kwargs):
if name == "applications":
return MagicMock(run=lambda *_a, **_k: [apps])
scrape = ScrapeTargetLookup()
scrape._templar = _Templar(variables)
scrape._loader = MagicMock()
return scrape
with (
patch(
"plugins.lookup.native_metrics_target.lookup_loader"
) as native_loader_mock,
patch("plugins.lookup.scrape_target.lookup_loader") as scrape_loader_mock,
patch(
"plugins.lookup.native_metrics_target.get_entity_name",
side_effect=lambda app_id: ENTITIES.get(app_id, ""),
),
patch(
"plugins.lookup.scrape_target.get_entity_name",
side_effect=lambda app_id: ENTITIES.get(app_id, ""),
),
):
native_loader_mock.get.side_effect = _dispatch
scrape_loader_mock.get.return_value = MagicMock(run=lambda *_a, **_k: [apps])
return lookup.run(terms, variables=variables)
class TestNativeMetricsTargetLookup(unittest.TestCase):
def test_swarm_uses_per_task_dns_not_the_container_name(self):
apps = _apps({"port": 9000, "service_key": "synapse"})
result = _run(["web-app-matrix"], mode="swarm", apps=apps)
self.assertEqual(result, ["tasks.matrix_synapse:9000"])
def test_compose_uses_the_container_name(self):
apps = _apps({"port": 9000, "service_key": "synapse"})
result = _run(["web-app-matrix"], mode="compose", apps=apps)
self.assertEqual(result, ["matrix-synapse:9000"])
def test_service_key_defaults_to_the_entity_name(self):
apps = {
"web-app-gitea": {
"services": {
"prometheus": {"native_metrics": {"port": 3000}},
"gitea": {
"name": "gitea",
"ports": {"internal": {"http": 3000}},
},
},
},
}
result = _run(["web-app-gitea"], mode="swarm", apps=apps)
self.assertEqual(result, ["tasks.gitea_gitea:3000"])
def test_native_metrics_port_takes_precedence_over_the_internal_port(self):
apps = _apps({"port": 9000, "service_key": "synapse"})
result = _run(["web-app-matrix"], mode="compose", apps=apps)
self.assertNotIn("8008", result[0])
def test_missing_port_raises(self):
apps = _apps({"service_key": "synapse"})
with self.assertRaises(AnsibleError):
_run(["web-app-matrix"], mode="compose", apps=apps)
def test_missing_application_id_raises(self):
with self.assertRaises(AnsibleError):
_run([], mode="compose", apps=_apps({"port": 9000}))
if __name__ == "__main__":
unittest.main()
tests/unit/python/roles/web-app-prometheus/__init__.py
View File
Whitespace-only changes.
tests/unit/python/roles/web-app-prometheus/test_blackbox_ready_module.py
+62
View File
@@ -0,0 +1,62 @@
from __future__ import annotations
import unittest
from jinja2 import Environment, FileSystemLoader, StrictUndefined, select_autoescape
from utils.cache.files import PROJECT_ROOT
from utils.cache.yaml import load_yaml_str
TEMPLATE_DIR = PROJECT_ROOT / "roles/web-app-prometheus/templates/configuration"
def ready_module(*, tls_mode: str, prometheus_tls: bool, ca_injected: bool) -> dict:
def lookup(name, *terms):
if name == "tls" and terms[1] == "enabled":
return prometheus_tls
if name == "ca_injected":
return ca_injected
raise AssertionError(f"the template calls an unexpected lookup: {name!r}")
env = Environment(
loader=FileSystemLoader(str(TEMPLATE_DIR)),
undefined=StrictUndefined,
autoescape=select_autoescape(),
)
env.filters["bool"] = bool
rendered = env.get_template("blackbox.yml.j2").render(
lookup=lookup,
application_id="web-app-prometheus",
TLS_MODE=tls_mode,
)
return load_yaml_str(rendered)["modules"]["http_ready"]["http"]
class TestBlackboxReadyModule(unittest.TestCase):
def test_an_http_target_passes_while_prometheus_itself_uses_tls(self):
module = ready_module(
tls_mode="letsencrypt", prometheus_tls=True, ca_injected=False
)
self.assertIs(module["fail_if_not_ssl"], False)
def test_certificates_are_verified_while_prometheus_itself_has_no_tls(self):
module = ready_module(
tls_mode="letsencrypt", prometheus_tls=False, ca_injected=False
)
self.assertIs(module["tls_config"]["insecure_skip_verify"], False)
def test_certificates_are_verified_once_the_private_ca_is_mounted(self):
module = ready_module(
tls_mode="self_signed", prometheus_tls=True, ca_injected=True
)
self.assertIs(module["tls_config"]["insecure_skip_verify"], False)
def test_verification_is_skipped_only_without_the_private_ca(self):
module = ready_module(
tls_mode="self_signed", prometheus_tls=False, ca_injected=False
)
self.assertIs(module["tls_config"]["insecure_skip_verify"], True)
if __name__ == "__main__":
unittest.main()
tests/unit/python/roles/web-app-prometheus/test_blackbox_targets.py
+84
View File
@@ -0,0 +1,84 @@
from __future__ import annotations
import unittest
from jinja2 import Environment, FileSystemLoader, StrictUndefined, select_autoescape
from utils.cache.files import PROJECT_ROOT
from utils.cache.yaml import load_yaml_str
TEMPLATE_DIR = PROJECT_ROOT / "roles/web-app-prometheus/templates/configuration"
ONION = "a" * 56 + ".onion"
DOMAINS = {
"web-app-prometheus": [f"prometheus.{ONION}"],
"web-app-nextcloud": {"nextcloud": f"next.cloud.{ONION}"},
"web-app-gitea": ["git.infinito.test"],
"web-svc-cdn": ["cdn.infinito.test"],
"svc-db-postgres": ["postgres.infinito.test"],
}
ROLES_WITH_THE_PROMETHEUS_SERVICE = ["web-app-nextcloud", "web-app-gitea"]
def lookup(name, *terms, **_kwargs):
if name == "domains":
return DOMAINS
if name == "roles_with_service":
return [{"id": role} for role in ROLES_WITH_THE_PROMETHEUS_SERVICE]
if name == "tls":
term, want = terms
scheme = "http" if term.endswith(".onion") else "https"
return {
"url.base": f"{scheme}://{term}/",
"protocols.web": "http",
"enabled": False,
}[want]
if name == "domain":
return DOMAINS[terms[0]][0]
if name == "scrape_target":
return f"{terms[1]}:9000"
if name == "native_metrics_apps":
return []
raise AssertionError(f"the template calls an unexpected lookup: {name!r}")
def probed_targets() -> dict[str, list[str]]:
env = Environment(
loader=FileSystemLoader(str(TEMPLATE_DIR)),
undefined=StrictUndefined,
autoescape=select_autoescape(),
)
env.filters["bool"] = bool
rendered = env.get_template("prometheus.yml.j2").render(
lookup=lookup,
application_id="web-app-prometheus",
CURRENT_PLAY_APPLICATIONS=list(DOMAINS),
DOMAIN_PRIMARY="infinito.test",
ALERT_RULES_CONFIG_DOCKER="/etc/prometheus/alert_rules.yml",
)
jobs = {job["job_name"]: job for job in load_yaml_str(rendered)["scrape_configs"]}
return {
group["labels"]["app"]: group["targets"]
for group in jobs["blackbox-healthz"]["static_configs"]
}
class TestBlackboxTargets(unittest.TestCase):
def test_an_onion_app_is_probed_under_its_onion_name_over_http(self):
self.assertEqual(
probed_targets()["web-app-nextcloud"],
[f"http://next.cloud.{ONION}/healthz/ready"],
)
def test_a_clearnet_app_is_probed_under_its_own_scheme(self):
self.assertEqual(
probed_targets()["web-app-gitea"],
["https://git.infinito.test/healthz/ready"],
)
def test_only_the_roles_the_service_lookup_names_are_probed(self):
self.assertEqual(set(probed_targets()), set(ROLES_WITH_THE_PROMETHEUS_SERVICE))
if __name__ == "__main__":
unittest.main()