From b8fb3522a49d83721a21c5dcc5b78cbab4231d36 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 02:01:46 +0200 Subject: [PATCH 01/13] Add cli/meta/applications/ressources resource aggregator Adds a CLI that reports per-container resource budgets (mem_reservation, mem_limit, pids_limit, cpus) for an Ansible role, recursively following enabled shared services via the service registry. Aggregation rules: mem_* and pids_limit are summed; cpus is MAX (shared across containers, not additive). Includes text + JSON output, cycle protection, and unit tests covering parsing helpers, aggregation, recursion, rendering, and end-to-end CLI invocation. Co-Authored-By: Claude Opus 4.7 --- cli/meta/applications/ressources/__init__.py | 0 cli/meta/applications/ressources/__main__.py | 384 +++++++++++++ .../meta/applications/ressources/__init__.py | 0 .../meta/applications/ressources/test_main.py | 516 ++++++++++++++++++ 4 files changed, 900 insertions(+) create mode 100644 cli/meta/applications/ressources/__init__.py create mode 100644 cli/meta/applications/ressources/__main__.py create mode 100644 tests/unit/cli/meta/applications/ressources/__init__.py create mode 100644 tests/unit/cli/meta/applications/ressources/test_main.py diff --git a/cli/meta/applications/ressources/__init__.py b/cli/meta/applications/ressources/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cli/meta/applications/ressources/__main__.py b/cli/meta/applications/ressources/__main__.py new file mode 100644 index 0000000000..a59338f454 --- /dev/null +++ b/cli/meta/applications/ressources/__main__.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python3 +# cli/meta/applications/ressources/__main__.py +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +REPO_ROOT = Path(__file__).resolve().parents[4] +sys.path.insert(0, str(REPO_ROOT)) + +from humanfriendly import format_size, parse_size # noqa: E402 + +from utils.entity_name_utils import get_entity_name # noqa: E402 +from utils.service_registry import ( # noqa: E402 + build_service_registry_from_applications, + load_applications_from_roles_dir, +) + + +ROLES_DIR = REPO_ROOT / "roles" + + +def _as_mapping(value: Any) -> Dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _parse_mem_bytes(value: Any) -> Optional[int]: + if value is None: + return None + if isinstance(value, (int, float)): + return int(value) + text = str(value).strip() + if not text: + return None + try: + return int(parse_size(text)) + except Exception: + return None + + +def _parse_cpus(value: Any) -> Optional[float]: + if value is None: + return None + if isinstance(value, (int, float)): + return float(value) + try: + return float(str(value).strip()) + except (TypeError, ValueError): + return None + + +def _parse_int(value: Any) -> Optional[int]: + if value is None: + return None + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + try: + return int(str(value).strip()) + except (TypeError, ValueError): + return None + + +def _is_enabled(service_conf: Dict[str, Any], is_primary: bool) -> bool: + if "enabled" not in service_conf: + return is_primary + raw = service_conf.get("enabled") + if isinstance(raw, bool): + return raw + text = str(raw).strip().lower() + if text in ("false", "0", "no", "off"): + return False + return True + + +def _is_shared(service_conf: Dict[str, Any]) -> bool: + raw = service_conf.get("shared", False) + if isinstance(raw, bool): + return raw + return str(raw).strip().lower() in ("true", "1", "yes", "on") + + +def _row_for_service( + role_name: str, + service_key: str, + service_conf: Dict[str, Any], +) -> Dict[str, Any]: + return { + "role": role_name, + "service": service_key, + "mem_reservation_raw": service_conf.get("mem_reservation"), + "mem_limit_raw": service_conf.get("mem_limit"), + "pids_limit_raw": service_conf.get("pids_limit"), + "cpus_raw": service_conf.get("cpus"), + "mem_reservation_bytes": _parse_mem_bytes(service_conf.get("mem_reservation")), + "mem_limit_bytes": _parse_mem_bytes(service_conf.get("mem_limit")), + "pids_limit_int": _parse_int(service_conf.get("pids_limit")), + "cpus_float": _parse_cpus(service_conf.get("cpus")), + } + + +def collect_role_resources( + role_name: str, + applications: Dict[str, Dict[str, Any]], + service_registry: Dict[str, Dict[str, Any]], + visited: set, + rows: List[Dict[str, Any]], + warnings: List[str], +) -> None: + if role_name in visited: + return + visited.add(role_name) + + if role_name not in applications: + warnings.append(f"role '{role_name}' has no config/main.yml; skipping") + return + + config = _as_mapping(applications[role_name]) + services = _as_mapping(_as_mapping(config.get("compose")).get("services")) + entity_name = get_entity_name(role_name) + + if entity_name and entity_name in services: + primary_conf = _as_mapping(services.get(entity_name)) + rows.append(_row_for_service(role_name, entity_name, primary_conf)) + else: + warnings.append( + f"role '{role_name}' has no compose.services.{entity_name or ''} entry" + ) + + shared_dependencies: List[str] = [] + for service_key, service_conf in services.items(): + if service_key == entity_name: + continue + service_conf = _as_mapping(service_conf) + if not service_conf: + continue + + if not _is_enabled(service_conf, is_primary=False): + continue + + if _is_shared(service_conf): + provider = _as_mapping(service_registry.get(service_key)) + provider_role = provider.get("role") if provider else None + if provider_role and provider_role != role_name: + shared_dependencies.append(provider_role) + elif not provider_role: + warnings.append( + f"{role_name}: shared service '{service_key}' has no registered provider" + ) + else: + rows.append(_row_for_service(role_name, service_key, service_conf)) + + for provider_role in shared_dependencies: + collect_role_resources( + provider_role, + applications, + service_registry, + visited, + rows, + warnings, + ) + + +def aggregate(rows: List[Dict[str, Any]]) -> Dict[str, Any]: + total_mem_res = 0 + total_mem_lim = 0 + total_pids = 0 + max_cpus = 0.0 + any_mem_res = any_mem_lim = any_pids = any_cpus = False + + for row in rows: + if row["mem_reservation_bytes"] is not None: + total_mem_res += row["mem_reservation_bytes"] + any_mem_res = True + if row["mem_limit_bytes"] is not None: + total_mem_lim += row["mem_limit_bytes"] + any_mem_lim = True + if row["pids_limit_int"] is not None: + total_pids += row["pids_limit_int"] + any_pids = True + if row["cpus_float"] is not None: + max_cpus = max(max_cpus, row["cpus_float"]) + any_cpus = True + + return { + "mem_reservation_bytes": total_mem_res if any_mem_res else None, + "mem_limit_bytes": total_mem_lim if any_mem_lim else None, + "pids_limit_int": total_pids if any_pids else None, + "cpus_float": max_cpus if any_cpus else None, + } + + +def _fmt_mem(value: Optional[int]) -> str: + if value is None: + return "-" + return format_size(value, binary=False) + + +def _fmt_int(value: Optional[int]) -> str: + return "-" if value is None else str(value) + + +def _fmt_float(value: Optional[float]) -> str: + if value is None: + return "-" + if float(value).is_integer(): + return str(int(value)) + return f"{value:g}" + + +def render_text( + role_name: str, + rows: List[Dict[str, Any]], + totals: Dict[str, Any], + warnings: List[str], +) -> str: + headers = ["service", "role", "mem_reservation", "mem_limit", "pids_limit", "cpus"] + table_rows: List[Tuple[str, ...]] = [] + + for row in sorted(rows, key=lambda r: (r["service"], r["role"])): + table_rows.append( + ( + row["service"], + row["role"], + _fmt_mem(row["mem_reservation_bytes"]), + _fmt_mem(row["mem_limit_bytes"]), + _fmt_int(row["pids_limit_int"]), + _fmt_float(row["cpus_float"]), + ) + ) + + total_label = "TOTAL (mem=SUM, pids=SUM max-provisioned, cpus=MAX)" + total_row = ( + total_label, + "", + _fmt_mem(totals["mem_reservation_bytes"]), + _fmt_mem(totals["mem_limit_bytes"]), + _fmt_int(totals["pids_limit_int"]), + _fmt_float(totals["cpus_float"]), + ) + + widths = [len(h) for h in headers] + for r in table_rows + [total_row]: + for i, cell in enumerate(r): + widths[i] = max(widths[i], len(cell)) + + def fmt_row(cells: Tuple[str, ...]) -> str: + return " ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells)) + + sep = " ".join("-" * w for w in widths) + lines = [ + f"# Resources for role: {role_name}", + "", + fmt_row(tuple(headers)), + sep, + ] + lines.extend(fmt_row(r) for r in table_rows) + lines.append(sep) + lines.append(fmt_row(total_row)) + + if warnings: + lines.append("") + lines.append("# Warnings") + for w in warnings: + lines.append(f"! {w}") + + return "\n".join(lines) + + +def render_json( + role_name: str, + rows: List[Dict[str, Any]], + totals: Dict[str, Any], + warnings: List[str], +) -> str: + def _row(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "role": row["role"], + "service": row["service"], + "mem_reservation": { + "raw": row["mem_reservation_raw"], + "bytes": row["mem_reservation_bytes"], + "human": _fmt_mem(row["mem_reservation_bytes"]), + }, + "mem_limit": { + "raw": row["mem_limit_raw"], + "bytes": row["mem_limit_bytes"], + "human": _fmt_mem(row["mem_limit_bytes"]), + }, + "pids_limit": { + "raw": row["pids_limit_raw"], + "value": row["pids_limit_int"], + }, + "cpus": { + "raw": row["cpus_raw"], + "value": row["cpus_float"], + }, + } + + payload = { + "role": role_name, + "services": [_row(r) for r in rows], + "totals": { + "mem_reservation": { + "bytes": totals["mem_reservation_bytes"], + "human": _fmt_mem(totals["mem_reservation_bytes"]), + }, + "mem_limit": { + "bytes": totals["mem_limit_bytes"], + "human": _fmt_mem(totals["mem_limit_bytes"]), + }, + "pids_limit": {"value": totals["pids_limit_int"]}, + "cpus": {"value": totals["cpus_float"]}, + "aggregation": { + "mem_reservation": "sum", + "mem_limit": "sum", + "pids_limit": "sum (max-provisioned; per-container cap, not shared load)", + "cpus": "max", + }, + }, + "warnings": warnings, + } + return json.dumps(payload, indent=2) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Compute aggregated container resources (mem_reservation, mem_limit, " + "pids_limit, cpus) for an Ansible role. Follows enabled shared services " + "recursively via the service registry. mem_reservation/mem_limit are " + "summed; pids_limit is summed as a max-provisioned host-pid budget " + "(per-container cap, not actual shared load); cpus is max (shared " + "across containers, not additive)." + ) + ) + parser.add_argument( + "--role", + required=True, + help="Role name (directory under roles/), e.g. web-app-peertube", + ) + parser.add_argument( + "-f", + "--format", + choices=("text", "json"), + default="text", + help="Output format (default: text).", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + applications = load_applications_from_roles_dir(ROLES_DIR) + service_registry = build_service_registry_from_applications(applications) + + rows: List[Dict[str, Any]] = [] + warnings: List[str] = [] + collect_role_resources( + role_name=args.role, + applications=applications, + service_registry=service_registry, + visited=set(), + rows=rows, + warnings=warnings, + ) + + totals = aggregate(rows) + + if args.format == "json": + print(render_json(args.role, rows, totals, warnings)) + else: + print(render_text(args.role, rows, totals, warnings)) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/cli/meta/applications/ressources/__init__.py b/tests/unit/cli/meta/applications/ressources/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/cli/meta/applications/ressources/test_main.py b/tests/unit/cli/meta/applications/ressources/test_main.py new file mode 100644 index 0000000000..ca3a824932 --- /dev/null +++ b/tests/unit/cli/meta/applications/ressources/test_main.py @@ -0,0 +1,516 @@ +from __future__ import annotations + +import io +import json +import unittest +from contextlib import redirect_stdout +from unittest.mock import patch + + +class TestParsingHelpers(unittest.TestCase): + def test_parse_mem_bytes_handles_various_units(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + self.assertEqual(cli._parse_mem_bytes("1g"), 1_000_000_000) + self.assertEqual(cli._parse_mem_bytes("512m"), 512_000_000) + self.assertEqual(cli._parse_mem_bytes("256M"), 256_000_000) + self.assertEqual(cli._parse_mem_bytes(1024), 1024) + self.assertIsNone(cli._parse_mem_bytes(None)) + self.assertIsNone(cli._parse_mem_bytes("")) + self.assertIsNone(cli._parse_mem_bytes("not-a-size")) + + def test_parse_cpus(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + self.assertEqual(cli._parse_cpus(4), 4.0) + self.assertEqual(cli._parse_cpus("2.0"), 2.0) + self.assertEqual(cli._parse_cpus("0.5"), 0.5) + self.assertIsNone(cli._parse_cpus(None)) + self.assertIsNone(cli._parse_cpus("abc")) + + def test_parse_int(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + self.assertEqual(cli._parse_int(2048), 2048) + self.assertEqual(cli._parse_int("1024"), 1024) + self.assertIsNone(cli._parse_int(None)) + self.assertIsNone(cli._parse_int(True)) + self.assertIsNone(cli._parse_int("not-int")) + + def test_is_enabled_primary_defaults_true_when_key_missing(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + self.assertTrue(cli._is_enabled({}, is_primary=True)) + self.assertFalse(cli._is_enabled({}, is_primary=False)) + + def test_is_enabled_respects_explicit_flag(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + self.assertTrue(cli._is_enabled({"enabled": True}, is_primary=False)) + self.assertFalse(cli._is_enabled({"enabled": False}, is_primary=True)) + self.assertFalse(cli._is_enabled({"enabled": "false"}, is_primary=True)) + self.assertFalse(cli._is_enabled({"enabled": "0"}, is_primary=True)) + + def test_is_enabled_treats_template_strings_as_truthy(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + self.assertTrue( + cli._is_enabled( + {"enabled": "{{ RECAPTCHA_ENABLED | bool }}"}, + is_primary=False, + ) + ) + + def test_is_shared(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + self.assertTrue(cli._is_shared({"shared": True})) + self.assertTrue(cli._is_shared({"shared": "true"})) + self.assertFalse(cli._is_shared({})) + self.assertFalse(cli._is_shared({"shared": False})) + + +class TestAggregate(unittest.TestCase): + def _row( + self, + mem_res: int | None = None, + mem_lim: int | None = None, + pids: int | None = None, + cpus: float | None = None, + ) -> dict: + return { + "role": "r", + "service": "s", + "mem_reservation_raw": None, + "mem_limit_raw": None, + "pids_limit_raw": None, + "cpus_raw": None, + "mem_reservation_bytes": mem_res, + "mem_limit_bytes": mem_lim, + "pids_limit_int": pids, + "cpus_float": cpus, + } + + def test_sums_mem_and_pids_and_takes_max_cpus(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + rows = [ + self._row(mem_res=1_000, mem_lim=2_000, pids=100, cpus=2.0), + self._row(mem_res=500, mem_lim=1_000, pids=50, cpus=4.0), + self._row(mem_res=250, mem_lim=500, pids=25, cpus=1.0), + ] + totals = cli.aggregate(rows) + self.assertEqual(totals["mem_reservation_bytes"], 1_750) + self.assertEqual(totals["mem_limit_bytes"], 3_500) + self.assertEqual(totals["pids_limit_int"], 175) + self.assertEqual(totals["cpus_float"], 4.0) + + def test_returns_none_when_all_values_missing(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + totals = cli.aggregate([self._row(), self._row()]) + self.assertIsNone(totals["mem_reservation_bytes"]) + self.assertIsNone(totals["mem_limit_bytes"]) + self.assertIsNone(totals["pids_limit_int"]) + self.assertIsNone(totals["cpus_float"]) + + def test_ignores_none_entries_for_individual_columns(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + rows = [ + self._row(mem_lim=1_000, cpus=2.0), + self._row(pids=10), + ] + totals = cli.aggregate(rows) + self.assertIsNone(totals["mem_reservation_bytes"]) + self.assertEqual(totals["mem_limit_bytes"], 1_000) + self.assertEqual(totals["pids_limit_int"], 10) + self.assertEqual(totals["cpus_float"], 2.0) + + +class TestCollectRoleResources(unittest.TestCase): + def _fake_registry(self) -> dict: + return { + "oidc": {"role": "web-app-keycloak"}, + "email": {"role": "web-app-mailu"}, + "postgres": {"role": "svc-db-postgres"}, + } + + def _fake_applications(self) -> dict: + return { + "web-app-peertube": { + "compose": { + "services": { + "peertube": { + "cpus": 4, + "mem_reservation": "4g", + "mem_limit": "8g", + "pids_limit": 2048, + }, + "redis": { + "enabled": True, + "cpus": "0.5", + "mem_reservation": "256m", + "mem_limit": "512m", + "pids_limit": 512, + }, + "oidc": {"enabled": True, "shared": True}, + "postgres": {"enabled": True, "shared": True}, + "email": {"enabled": True, "shared": True}, + "css": {"enabled": False, "shared": True}, + } + } + }, + "web-app-keycloak": { + "compose": { + "services": { + "keycloak": { + "cpus": "2.0", + "mem_reservation": "2g", + "mem_limit": "4g", + "pids_limit": 1024, + }, + } + } + }, + "web-app-mailu": { + "compose": { + "services": { + "mailu": {}, + "oidc": {"enabled": True, "shared": True}, + } + } + }, + "svc-db-postgres": { + "compose": { + "services": { + "postgres": { + "cpus": 2, + "mem_reservation": "4g", + "mem_limit": "6g", + "pids_limit": 1024, + } + } + } + }, + } + + def test_primary_and_sidecar_grouped_then_shared_deps(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + rows: list = [] + warnings: list = [] + cli.collect_role_resources( + role_name="web-app-peertube", + applications=self._fake_applications(), + service_registry=self._fake_registry(), + visited=set(), + rows=rows, + warnings=warnings, + ) + labels = [(r["role"], r["service"]) for r in rows] + self.assertEqual(labels[0], ("web-app-peertube", "peertube")) + self.assertEqual(labels[1], ("web-app-peertube", "redis")) + self.assertIn(("web-app-keycloak", "keycloak"), labels) + self.assertIn(("svc-db-postgres", "postgres"), labels) + self.assertIn(("web-app-mailu", "mailu"), labels) + peertube_pos = labels.index(("web-app-peertube", "redis")) + keycloak_pos = labels.index(("web-app-keycloak", "keycloak")) + self.assertLess(peertube_pos, keycloak_pos) + + def test_disabled_services_are_skipped(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + rows: list = [] + cli.collect_role_resources( + role_name="web-app-peertube", + applications=self._fake_applications(), + service_registry=self._fake_registry(), + visited=set(), + rows=rows, + warnings=[], + ) + labels = {(r["role"], r["service"]) for r in rows} + self.assertNotIn(("web-app-peertube", "css"), labels) + + def test_cycle_protection_visits_each_role_once(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + # mailu declares oidc (-> keycloak). keycloak could declare email in + # theory. Build a cycle keycloak <-> mailu and ensure no infinite loop. + apps = self._fake_applications() + apps["web-app-keycloak"]["compose"]["services"]["email"] = { + "enabled": True, + "shared": True, + } + + rows: list = [] + cli.collect_role_resources( + role_name="web-app-peertube", + applications=apps, + service_registry=self._fake_registry(), + visited=set(), + rows=rows, + warnings=[], + ) + roles_in_rows = [r["role"] for r in rows] + self.assertEqual(roles_in_rows.count("web-app-keycloak"), 1) + self.assertEqual(roles_in_rows.count("web-app-mailu"), 1) + + def test_warns_when_shared_service_has_no_provider(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + apps = { + "web-app-x": { + "compose": { + "services": { + "x": {"cpus": 1}, + "unknown": {"enabled": True, "shared": True}, + } + } + } + } + warnings: list = [] + cli.collect_role_resources( + role_name="web-app-x", + applications=apps, + service_registry={}, + visited=set(), + rows=[], + warnings=warnings, + ) + self.assertTrue(any("unknown" in w for w in warnings)) + + def test_warns_when_role_config_missing(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + warnings: list = [] + cli.collect_role_resources( + role_name="missing-role", + applications={}, + service_registry={}, + visited=set(), + rows=[], + warnings=warnings, + ) + self.assertTrue(any("missing-role" in w for w in warnings)) + + def test_shared_entry_without_registered_provider_does_not_recurse( + self, + ) -> None: + from cli.meta.applications.ressources import __main__ as cli + + apps = { + "web-app-y": { + "compose": { + "services": { + "y": {"cpus": 1, "mem_limit": "100m"}, + "ghost": {"enabled": True, "shared": True}, + } + } + } + } + rows: list = [] + cli.collect_role_resources( + role_name="web-app-y", + applications=apps, + service_registry={}, + visited=set(), + rows=rows, + warnings=[], + ) + self.assertEqual( + [(r["role"], r["service"]) for r in rows], [("web-app-y", "y")] + ) + + +class TestRenderText(unittest.TestCase): + def test_sorts_by_service_then_role_and_uses_labeled_total(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + rows = [ + { + "role": "web-app-peertube", + "service": "redis", + "mem_reservation_raw": "256m", + "mem_limit_raw": "512m", + "pids_limit_raw": 512, + "cpus_raw": "0.5", + "mem_reservation_bytes": 256_000_000, + "mem_limit_bytes": 512_000_000, + "pids_limit_int": 512, + "cpus_float": 0.5, + }, + { + "role": "web-app-mailu", + "service": "redis", + "mem_reservation_raw": "256m", + "mem_limit_raw": "512m", + "pids_limit_raw": 256, + "cpus_raw": "0.2", + "mem_reservation_bytes": 256_000_000, + "mem_limit_bytes": 512_000_000, + "pids_limit_int": 256, + "cpus_float": 0.2, + }, + { + "role": "web-app-peertube", + "service": "peertube", + "mem_reservation_raw": "4g", + "mem_limit_raw": "8g", + "pids_limit_raw": 2048, + "cpus_raw": 4, + "mem_reservation_bytes": 4_000_000_000, + "mem_limit_bytes": 8_000_000_000, + "pids_limit_int": 2048, + "cpus_float": 4.0, + }, + ] + totals = cli.aggregate(rows) + text = cli.render_text("web-app-peertube", rows, totals, warnings=[]) + + lines = text.splitlines() + data_lines = [ln for ln in lines if ln and not ln.startswith(("#", "-"))] + # First body line is header, then sorted data rows. + self.assertTrue(data_lines[0].startswith("service")) + self.assertTrue(data_lines[1].startswith("peertube")) + # The two redis rows sort after peertube, mailu before peertube by role. + self.assertIn("redis", data_lines[2]) + self.assertIn("web-app-mailu", data_lines[2]) + self.assertIn("redis", data_lines[3]) + self.assertIn("web-app-peertube", data_lines[3]) + # Total row uses the labeled form. + self.assertTrue( + any( + "TOTAL (mem=SUM, pids=SUM max-provisioned, cpus=MAX)" in ln + for ln in lines + ) + ) + + def test_appends_warnings_section(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + text = cli.render_text( + role_name="web-app-x", + rows=[], + totals=cli.aggregate([]), + warnings=["shared service 'foo' has no registered provider"], + ) + self.assertIn("# Warnings", text) + self.assertIn("! shared service 'foo' has no registered provider", text) + + +class TestRenderJson(unittest.TestCase): + def test_emits_services_totals_warnings_and_aggregation_metadata(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + rows = [ + { + "role": "web-app-peertube", + "service": "peertube", + "mem_reservation_raw": "4g", + "mem_limit_raw": "8g", + "pids_limit_raw": 2048, + "cpus_raw": 4, + "mem_reservation_bytes": 4_000_000_000, + "mem_limit_bytes": 8_000_000_000, + "pids_limit_int": 2048, + "cpus_float": 4.0, + } + ] + totals = cli.aggregate(rows) + payload = json.loads( + cli.render_json("web-app-peertube", rows, totals, warnings=["w"]) + ) + self.assertEqual(payload["role"], "web-app-peertube") + self.assertEqual(len(payload["services"]), 1) + self.assertEqual(payload["totals"]["mem_limit"]["bytes"], 8_000_000_000) + self.assertEqual(payload["totals"]["cpus"]["value"], 4.0) + self.assertEqual(payload["totals"]["aggregation"]["cpus"], "max") + self.assertTrue( + payload["totals"]["aggregation"]["pids_limit"].startswith("sum") + ) + self.assertEqual(payload["warnings"], ["w"]) + + +class TestCliMain(unittest.TestCase): + def test_text_format_runs_end_to_end(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + fake_apps = { + "web-app-x": { + "compose": { + "services": { + "x": { + "cpus": 1, + "mem_reservation": "100m", + "mem_limit": "200m", + "pids_limit": 64, + } + } + } + } + } + + with ( + patch.object( + cli, "load_applications_from_roles_dir", return_value=fake_apps + ), + patch.object( + cli, "build_service_registry_from_applications", return_value={} + ), + patch.object(cli.sys, "argv", ["prog", "--role", "web-app-x"]), + ): + out = io.StringIO() + with redirect_stdout(out): + rc = cli.main() + + self.assertEqual(rc, 0) + output = out.getvalue() + self.assertIn("web-app-x", output) + self.assertIn("TOTAL", output) + + def test_json_format_runs_end_to_end(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + fake_apps = { + "web-app-x": { + "compose": { + "services": { + "x": { + "cpus": 2, + "mem_reservation": "1g", + "mem_limit": "2g", + "pids_limit": 128, + } + } + } + } + } + + with ( + patch.object( + cli, "load_applications_from_roles_dir", return_value=fake_apps + ), + patch.object( + cli, "build_service_registry_from_applications", return_value={} + ), + patch.object( + cli.sys, + "argv", + ["prog", "--role", "web-app-x", "--format", "json"], + ), + ): + out = io.StringIO() + with redirect_stdout(out): + rc = cli.main() + + self.assertEqual(rc, 0) + payload = json.loads(out.getvalue()) + self.assertEqual(payload["role"], "web-app-x") + self.assertEqual(payload["totals"]["mem_limit"]["bytes"], 2_000_000_000) + self.assertEqual(payload["totals"]["cpus"]["value"], 2.0) + + +if __name__ == "__main__": + unittest.main() From 1c85dec2b5bffb418feff999e988b359876ae2e2 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 02:17:52 +0200 Subject: [PATCH 02/13] Add realistic resource caps to shared and mailu services Fills in mem_reservation/mem_limit/pids_limit/cpus on service entries that previously had none, so the role resource aggregator reports a realistic per-role footprint. Sets light caps on static/SPA providers (asset, cdn, file, dashboard) and flips mailu sub-services (admin, imap, smtp, antispam, antivirus, oletools, webdav, fetchmail, webmail, resolver, front) to enabled: true so the existing values are picked up during the shared-service walk. Co-Authored-By: Claude Opus 4.7 --- roles/web-app-dashboard/config/main.yml | 4 ++++ roles/web-app-mailu/config/main.yml | 11 +++++++++++ roles/web-svc-asset/config/main.yml | 4 ++++ roles/web-svc-cdn/config/main.yml | 4 ++++ roles/web-svc-file/config/main.yml | 4 ++++ 5 files changed, 27 insertions(+) diff --git a/roles/web-app-dashboard/config/main.yml b/roles/web-app-dashboard/config/main.yml index 1b2b42412a..17e1084efa 100644 --- a/roles/web-app-dashboard/config/main.yml +++ b/roles/web-app-dashboard/config/main.yml @@ -52,6 +52,10 @@ compose: image: "ghcr.io/kevinveenbirkenbach/port-ui" version: "1.1.0" min_storage: "1GB" + cpus: "0.3" + mem_reservation: "256m" + mem_limit: "512m" + pids_limit: 256 css: enabled: true shared: true diff --git a/roles/web-app-mailu/config/main.yml b/roles/web-app-mailu/config/main.yml index 742f33d566..330fc95ca5 100644 --- a/roles/web-app-mailu/config/main.yml +++ b/roles/web-app-mailu/config/main.yml @@ -61,56 +61,67 @@ compose: mem_limit: "2g" pids_limit: 512 admin: + enabled: true cpus: "0.3" mem_reservation: "512m" mem_limit: "1g" pids_limit: 512 imap: + enabled: true cpus: "0.5" mem_reservation: "1g" mem_limit: "1.5g" pids_limit: 512 smtp: + enabled: true cpus: "0.5" mem_reservation: "1g" mem_limit: "1.5g" pids_limit: 512 antispam: + enabled: true cpus: "0.6" mem_reservation: "1g" mem_limit: "1.5g" pids_limit: 512 antivirus: + enabled: true cpus: "0.6" mem_reservation: "2g" mem_limit: "3g" pids_limit: 512 oletools: + enabled: true cpus: "0.2" mem_reservation: "256m" mem_limit: "512m" pids_limit: 256 webdav: + enabled: true cpus: "0.2" mem_reservation: "256m" mem_limit: "512m" pids_limit: 256 fetchmail: + enabled: true cpus: "0.2" mem_reservation: "256m" mem_limit: "512m" pids_limit: 256 webmail: + enabled: true cpus: "0.3" mem_reservation: "512m" mem_limit: "1g" pids_limit: 512 resolver: + enabled: true cpus: "0.2" mem_reservation: "256m" mem_limit: "512m" pids_limit: 256 front: + enabled: true cpus: "0.3" mem_reservation: "512m" mem_limit: "1g" diff --git a/roles/web-svc-asset/config/main.yml b/roles/web-svc-asset/config/main.yml index 0957ae26cd..0514ab1fde 100644 --- a/roles/web-svc-asset/config/main.yml +++ b/roles/web-svc-asset/config/main.yml @@ -10,6 +10,10 @@ compose: asset: enabled: false shared: true + cpus: "0.2" + mem_reservation: "128m" + mem_limit: "256m" + pids_limit: 128 # Assets are published through the shared file host under /assets. # Keep the file dependency here so consumers only depend on web-svc-asset. file: diff --git a/roles/web-svc-cdn/config/main.yml b/roles/web-svc-cdn/config/main.yml index 3641817092..99fe43fc74 100644 --- a/roles/web-svc-cdn/config/main.yml +++ b/roles/web-svc-cdn/config/main.yml @@ -10,6 +10,10 @@ compose: cdn: enabled: false shared: true + cpus: "0.2" + mem_reservation: "128m" + mem_limit: "256m" + pids_limit: 128 matomo: enabled: true shared: true diff --git a/roles/web-svc-file/config/main.yml b/roles/web-svc-file/config/main.yml index 38a7c22bdb..34869d58b5 100644 --- a/roles/web-svc-file/config/main.yml +++ b/roles/web-svc-file/config/main.yml @@ -8,6 +8,10 @@ compose: file: enabled: false shared: true + cpus: "0.2" + mem_reservation: "128m" + mem_limit: "256m" + pids_limit: 128 dashboard: enabled: true shared: true From 7dc55d49da5319a119654c1e1d7106d9f7355281 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 02:21:15 +0200 Subject: [PATCH 03/13] Skip toggle-only service entries in resource aggregator Adds a container-shaped heuristic so non-shared compose.services entries that declare neither resource fields (mem_*/pids_limit/cpus) nor container-identifying fields (image/name/version/container) are treated as feature flags and excluded from the resource table. Keeps the keycloak recaptcha toggle and similar booleans out of the per-role footprint. Adds unit coverage for the helper and recursion behavior. Co-Authored-By: Claude Opus 4.7 --- cli/meta/applications/ressources/__main__.py | 10 ++++++ .../meta/applications/ressources/test_main.py | 36 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/cli/meta/applications/ressources/__main__.py b/cli/meta/applications/ressources/__main__.py index a59338f454..9329834da1 100644 --- a/cli/meta/applications/ressources/__main__.py +++ b/cli/meta/applications/ressources/__main__.py @@ -84,6 +84,14 @@ def _is_shared(service_conf: Dict[str, Any]) -> bool: return str(raw).strip().lower() in ("true", "1", "yes", "on") +_RESOURCE_KEYS = ("mem_reservation", "mem_limit", "pids_limit", "cpus") +_CONTAINER_KEYS = ("image", "name", "version", "container") + + +def _looks_like_container(service_conf: Dict[str, Any]) -> bool: + return any(key in service_conf for key in _RESOURCE_KEYS + _CONTAINER_KEYS) + + def _row_for_service( role_name: str, service_key: str, @@ -152,6 +160,8 @@ def collect_role_resources( f"{role_name}: shared service '{service_key}' has no registered provider" ) else: + if not _looks_like_container(service_conf): + continue rows.append(_row_for_service(role_name, service_key, service_conf)) for provider_role in shared_dependencies: diff --git a/tests/unit/cli/meta/applications/ressources/test_main.py b/tests/unit/cli/meta/applications/ressources/test_main.py index ca3a824932..8c85095017 100644 --- a/tests/unit/cli/meta/applications/ressources/test_main.py +++ b/tests/unit/cli/meta/applications/ressources/test_main.py @@ -69,6 +69,16 @@ class TestParsingHelpers(unittest.TestCase): self.assertFalse(cli._is_shared({})) self.assertFalse(cli._is_shared({"shared": False})) + def test_looks_like_container(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + self.assertTrue(cli._looks_like_container({"mem_limit": "256m"})) + self.assertTrue(cli._looks_like_container({"cpus": 1})) + self.assertTrue(cli._looks_like_container({"image": "alpine"})) + self.assertTrue(cli._looks_like_container({"name": "foo", "version": "1"})) + self.assertFalse(cli._looks_like_container({})) + self.assertFalse(cli._looks_like_container({"enabled": True})) + class TestAggregate(unittest.TestCase): def _row( @@ -281,6 +291,32 @@ class TestCollectRoleResources(unittest.TestCase): ) self.assertTrue(any("unknown" in w for w in warnings)) + def test_toggle_only_local_entries_are_skipped(self) -> None: + from cli.meta.applications.ressources import __main__ as cli + + apps = { + "web-app-x": { + "compose": { + "services": { + "x": {"cpus": 1, "mem_limit": "100m"}, + "feature_flag": {"enabled": True}, + } + } + } + } + rows: list = [] + cli.collect_role_resources( + role_name="web-app-x", + applications=apps, + service_registry={}, + visited=set(), + rows=rows, + warnings=[], + ) + services_in_rows = {r["service"] for r in rows} + self.assertIn("x", services_in_rows) + self.assertNotIn("feature_flag", services_in_rows) + def test_warns_when_role_config_missing(self) -> None: from cli.meta.applications.ressources import __main__ as cli From 64617514ac99fd32d4baf0c0ef89cb7fd6c4b4bd Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 02:38:54 +0200 Subject: [PATCH 04/13] Add OOM reproduction knobs to compose.yml with SPOT docs Expose INFINITO_MEM_LIMIT/INFINITO_MEMSWAP_LIMIT/INFINITO_CPUS on the infinito service so operators can cap local runs to GitHub-hosted runner specs and reproduce OOM failures like #162. Document the compose.yml as a dev/CI-only artefact under docs/contributing/artefact/files/ and link to it from the CI debugging guide. Co-Authored-By: Claude Opus 4.7 --- compose.yml | 7 ++ docs/contributing/actions/debugging/ci.md | 40 ++++++++++ .../artefact/files/compose.yml.md | 78 +++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 docs/contributing/artefact/files/compose.yml.md diff --git a/compose.yml b/compose.yml index d4d3b4b225..f6e1c4afda 100644 --- a/compose.yml +++ b/compose.yml @@ -68,6 +68,13 @@ services: container_name: infinito_nexus_${INFINITO_DISTRO:-debian} restart: "no" profiles: ["ci"] + # Resource caps for reproducing OOM/OOM-kill scenarios locally (e.g. issue #162 + # web-app-peertube plugin install rc=137). Defaults of 0 map to "unlimited" + # in the Docker engine, so leaving these env vars unset behaves as if the + # keys were omitted. + mem_limit: ${INFINITO_MEM_LIMIT:-0} + memswap_limit: ${INFINITO_MEMSWAP_LIMIT:-0} + cpus: ${INFINITO_CPUS:-0} <<: *ci_defaults volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw diff --git a/docs/contributing/actions/debugging/ci.md b/docs/contributing/actions/debugging/ci.md index eeda790e99..4386ee208a 100644 --- a/docs/contributing/actions/debugging/ci.md +++ b/docs/contributing/actions/debugging/ci.md @@ -27,3 +27,43 @@ Prefer the manual workflow in [entry-manual.yml](../../../../.github/workflows/e This gives faster feedback and protects shared CI runners. More information [here](https://hub.infinito.nexus/t/infinito-nexus-ci-cd-debugging-guide/462). + +## Reproducing Runner Resource Limits Locally ๐Ÿ’พ + +Some CI failures โ€” most notably OOM-kills such as [#162](https://github.com/infinito-nexus/core/issues/162) (PeerTube plugin install, `rc=137`) โ€” only surface under the memory and CPU ceiling of the GitHub-hosted runner and cannot be reproduced on a workstation with plenty of RAM. To reproduce these failures locally, you MUST cap the resources of the top-level `infinito` service in [compose.yml](../../../../compose.yml) to values that match (or undercut) the runner specs below. + +### GitHub-Hosted Runner Specs + +All [workflows](../../../../.github/workflows/) target `runs-on: ubuntu-latest`, which maps to the GitHub-hosted standard Linux runner. The effective ceiling depends on the repository's visibility and plan: + +| Repository / plan | vCPU | RAM | SSD | +|------------------------------------------|------|--------|-------| +| Public repo (this repo, `ubuntu-latest`) | 4 | 16 GB | 14 GB | +| Private repo on Free / Pro / Team | 2 | 7 GB | 14 GB | +| Private repo on Enterprise | 4 | 16 GB | 14 GB | + +The effective budget for the actual workload is lower than the table suggests because the runner itself plus the nested `infinito` container (systemd, nested Docker, Ansible) reserve a non-trivial baseline. Plan for roughly **1โ€“2 GB of overhead** before any role deploys a service. + +For the authoritative specs, see the upstream docs for [standard hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories) and [larger runners](https://docs.github.com/en/actions/using-github-hosted-runners/using-larger-runners). + +### Env-Var Knobs + +[compose.yml](../../../../compose.yml) exposes three env vars on the `infinito` service that cap its memory, swap, and CPU ceiling. For the authoritative variable reference (names, defaults, purpose) see the SPOT in [compose.yml.md](../../artefact/files/compose.yml.md). This page SHOULD NOT duplicate that table and MUST link back to it when a contributor needs the precise definitions. + +### Canonical Reproduction Profiles + +Pick the profile that matches the runner you want to mimic: + +```sh +# Public repo runner (ubuntu-latest, 4 vCPU / 16 GB) โ€” the default CI target +INFINITO_MEM_LIMIT=16g INFINITO_MEMSWAP_LIMIT=16g INFINITO_CPUS=4 make + +# Private repo runner on Free/Pro/Team (2 vCPU / 7 GB) +INFINITO_MEM_LIMIT=7g INFINITO_MEMSWAP_LIMIT=7g INFINITO_CPUS=2 make + +# Aggressive OOM reproducer โ€” forces the memory spike that surfaces #162 +INFINITO_MEM_LIMIT=4g INFINITO_MEMSWAP_LIMIT=4g INFINITO_CPUS=2 make +``` + +Verify the caps applied with `docker compose -f compose.yml --profile ci config | grep -E 'mem_limit|memswap_limit|cpus'` before starting the deploy. + diff --git a/docs/contributing/artefact/files/compose.yml.md b/docs/contributing/artefact/files/compose.yml.md new file mode 100644 index 0000000000..f5b62ac2e1 --- /dev/null +++ b/docs/contributing/artefact/files/compose.yml.md @@ -0,0 +1,78 @@ +# `compose.yml` ๐Ÿณ + +This page is the SPOT for rules that govern the top-level [compose.yml](../../../../compose.yml) and for the environment variables it consumes. +For general documentation rules (links, writing, RFC 2119 keywords), see [documentation.md](../../documentation.md). +For CI debugging workflows that use these variables, see [ci.md](../../actions/debugging/ci.md). + +## Purpose ๐ŸŽฏ + +The top-level `compose.yml` is scoped to **local development and CI testing only**. +It MUST NOT be used to deploy or run a live Infinito.Nexus stack. +Production deployments MUST go through the Ansible roles and their own rendered compose files under `roles//templates/`. + +Within its scope, `compose.yml` MUST stay a thin orchestration surface for the `infinito` runner container and its CoreDNS sidecar. +It MUST NOT host application roles, fixtures, or test harnesses. + +## Structure ๐Ÿ“ + +- You MUST keep every service in `compose.yml` behind a `profiles:` entry so that `docker compose up` without a profile stays a no-op. +- You MUST expose tunable runtime parameters as env vars with safe defaults inline in the compose key (e.g. `${VAR:-default}`). +- You MUST NOT hardcode values that differ between operators (paths, image tags, CPU or memory caps). +- You SHOULD annotate non-obvious keys with a short inline comment that explains why they exist, not what they do. +- You MUST document every new env var in the table below when you add it to `compose.yml`. + +## Environment Variables ๐Ÿ“‹ + +All variables consumed by [compose.yml](../../../../compose.yml). Variables without a default are REQUIRED when the `ci` profile is active. + +### Image and Runtime + +| Variable | Default | Purpose | +|--------------------------|-----------------|-------------------------------------------------------------------------| +| `INFINITO_IMAGE` | none (required) | Image reference used by the `infinito` service. | +| `INFINITO_PULL_POLICY` | `never` | Compose `pull_policy`. Keep `never` for local builds, `always` for CI. | +| `INFINITO_DISTRO` | `debian` | Distro suffix appended to the `container_name`. | +| `INFINITO_COMPILE` | `1` | Passed into the container; toggles in-container compilation steps. | +| `NIX_CONFIG` | none | Build-arg forwarded to the Dockerfile for Nix configuration. | + +### Resource Caps (OOM Reproduction) + +These variables cap the `infinito` container so local runs can reproduce GitHub-hosted runner pressure. +Each default is `0`, which Docker interprets as "unlimited" and is identical to omitting the key. +See [ci.md](../../actions/debugging/ci.md) for runner specs and reproduction profiles. + +| Variable | Compose key | Default | Example | +|--------------------------|------------------|---------|---------| +| `INFINITO_MEM_LIMIT` | `mem_limit` | `0` | `16g` | +| `INFINITO_MEMSWAP_LIMIT` | `memswap_limit` | `0` | `16g` | +| `INFINITO_CPUS` | `cpus` | `0` | `4` | + +You SHOULD set `INFINITO_MEMSWAP_LIMIT` to the same value as `INFINITO_MEM_LIMIT` to disable swap inflation. +Allowing swap masks OOM conditions that the real runner would hit. + +### Storage + +| Variable | Default | Purpose | +|---------------------------|--------------------|------------------------------------------------------------------| +| `INFINITO_DOCKER_VOLUME` | `docker` | Named volume (or host path) backing the nested Docker directory. | +| `INFINITO_DOCKER_MOUNT` | `/var/lib/docker` | Mount point inside the container for the nested Docker data. | + +### Networking + +| Variable | Default | Purpose | +|-------------------------------|------------------|------------------------------------------------------------------------| +| `DNS_IP` | none (required) | Static IP assigned to the CoreDNS sidecar and used as the DNS server. | +| `IP4` | none (required) | Static IPv4 address assigned to the `infinito` container. | +| `DOMAIN` | none (required) | Base domain exported into the container environment. | +| `BIND_IP` | `127.0.0.1` | Host address that all published ports bind to. | +| `SUBNET` | `172.30.0.0/24` | IPAM subnet for the default bridge network. | +| `GATEWAY` | `172.30.0.1` | IPAM gateway for the default bridge network. | +| `INFINITO_OUTER_NETWORK_MTU` | `1500` | MTU for the bridge network. Lower when the host network is tunneled. | + +## Adding a Variable โž• + +When you introduce a new env var in `compose.yml`, you MUST: + +1. Use the `${VAR:-default}` form if a sensible default exists. If no default is safe, use `${VAR}` and treat the variable as REQUIRED. +2. Add a row to the matching table above with default and purpose. +3. Cross-link the variable from the relevant workflow page when the variable only exists to drive a specific workflow (e.g. resource caps link to [ci.md](../../actions/debugging/ci.md)). From 69e23ca35682aa9e5263b16f2da47feafcdf53d0 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 02:46:30 +0200 Subject: [PATCH 05/13] Derive peertube plugin-install heap cap from container mem_limit Replace the hardcoded 1024 MB ceiling for PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB with a dynamic value via node_max_old_space_size (pct=0.125, min=256, hardcap=1024). The plugin installer runs inside the peertube cgroup via container exec, so its budget is a share of the parent container, not a separate service claim. Keeps the install heap well below the runtime heap to leave headroom for npm's forked workers (node-gyp, postinstall) on constrained CI runners (#162). Co-Authored-By: Claude Opus 4.7 --- roles/web-app-peertube/config/main.yml | 2 +- roles/web-app-peertube/vars/main.yml | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/roles/web-app-peertube/config/main.yml b/roles/web-app-peertube/config/main.yml index 60bccb8297..4bb0e4cfdd 100644 --- a/roles/web-app-peertube/config/main.yml +++ b/roles/web-app-peertube/config/main.yml @@ -48,7 +48,7 @@ compose: postgres: enabled: true shared: true - peertube: + peertube: name: "peertube" version: "production-bookworm" image: "chocobozzz/peertube" diff --git a/roles/web-app-peertube/vars/main.yml b/roles/web-app-peertube/vars/main.yml index fc20415c18..8f18c782e0 100644 --- a/roles/web-app-peertube/vars/main.yml +++ b/roles/web-app-peertube/vars/main.yml @@ -19,8 +19,12 @@ PEERTUBE_MAX_OLD_SPACE_SIZE: "{{ applications | node_max_old_space_size(app _peertube_concurrency_candidate: "{{ ((PEERTUBE_CPUS | float) * 0.5) | round(0, 'floor') | int }}" PEERTUBE_TRANSCODING_CONCURRENCY: "{{ [ ( [ (_peertube_concurrency_candidate | int), 1 ] | max ), 8 ] | min }}" -# Heap cap for the one-off `npm run plugin:install` exec. Kept tight so the -# plugin install does not blow past the host RAM budget on constrained CI -# runners (where the container-wide NODE_OPTIONS would otherwise grant several -# GB of V8 heap and trip the host OOM killer). -PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB: 1024 +# Heap cap for the one-off `npm run plugin:install` exec. Derived relative to +# the peertube container's mem_limit (the installer runs inside that cgroup +# via `container exec`, so its budget is a share of the parent, not a +# separate service claim). pct=0.125 keeps the install heap well below the +# runtime heap (PEERTUBE_MAX_OLD_SPACE_SIZE uses 35%) because npm forks +# additional workers (node-gyp, postinstall) that each consume memory; hard- +# capped at 1024 MB to avoid tripping the host OOM-killer on constrained CI +# runners (see issue #162). +PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB: "{{ applications | node_max_old_space_size(application_id, entity_name, pct=0.125, min_mb=256, hardcap_mb=1024) }}" From 8cdfcad9ef3ce068347ca5f5397a2d57899d874e Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 03:18:45 +0200 Subject: [PATCH 06/13] Enlarge CI runner swap dynamically across deploy workflows Introduce scripts/github/enlarge_swap.sh that consumes all remaining space on / minus a 25 GB buffer (Docker layers, checkout, runtime state), falling back to /mnt when / is too small. Replaces the default ~4 GB runner swap so transient memory spikes (e.g. peertube plugin install, issue #162) don't trip the host OOM-killer on the 16 GB GitHub-hosted runner. Wire the script into all four deploy test workflows (local, universal, server, workstation) and flip free-disk-space to the aggressive config (tool-cache, large-packages, docker-images, swap-storage all true): the deploy actually runs inside the infinito container, so host-side toolchains, language runtimes, and the default Docker layer cache are dead weight. Update the workflow SPOT doc accordingly. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/test-deploy-local.yml | 21 ++++++++ .github/workflows/test-deploy-server.yml | 15 ++++-- .github/workflows/test-deploy-universal.yml | 15 ++++-- .github/workflows/test-deploy-workstation.yml | 15 ++++-- .../artefact/files/github/workflow.md | 41 +++++++++++++--- scripts/github/enlarge_swap.sh | 48 +++++++++++++++++++ 6 files changed, 136 insertions(+), 19 deletions(-) create mode 100755 scripts/github/enlarge_swap.sh diff --git a/.github/workflows/test-deploy-local.yml b/.github/workflows/test-deploy-local.yml index d236438540..0f5df083d0 100644 --- a/.github/workflows/test-deploy-local.yml +++ b/.github/workflows/test-deploy-local.yml @@ -88,8 +88,29 @@ jobs: apps: ${{ fromJson(needs.discover.outputs.apps) }} steps: + # Reclaim runner space before Docker pulls start. Deploy jobs otherwise + # compete with the default Ubuntu image payload for limited root space. + # Option rationale: docs/contributing/artefact/files/github/workflow.md + - name: Free disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - uses: actions/checkout@v6 + # Replace the default ~4 GB runner swap with a larger file so transient + # memory spikes (e.g. peertube plugin install, issue #162) don't trip + # the host OOM-killer. Script picks / or /mnt based on live free space. + - name: Enlarge swap space + shell: bash + run: ./scripts/github/enlarge_swap.sh + - name: Install deps (jq + envsubst) shell: bash run: ./scripts/install/apt.sh jq gettext-base diff --git a/.github/workflows/test-deploy-server.yml b/.github/workflows/test-deploy-server.yml index db83fe2138..e7ae1409ed 100644 --- a/.github/workflows/test-deploy-server.yml +++ b/.github/workflows/test-deploy-server.yml @@ -82,16 +82,23 @@ jobs: - name: Free disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 with: - tool-cache: false + tool-cache: true android: true dotnet: true haskell: true - large-packages: false - docker-images: false - swap-storage: false + large-packages: true + docker-images: true + swap-storage: true - uses: actions/checkout@v6 + # Replace the default ~4 GB runner swap with a larger file so transient + # memory spikes (e.g. peertube plugin install, issue #162) don't trip + # the host OOM-killer. Script picks / or /mnt based on live free space. + - name: Enlarge swap space + shell: bash + run: ./scripts/github/enlarge_swap.sh + - name: Resolve repository owner shell: bash run: | diff --git a/.github/workflows/test-deploy-universal.yml b/.github/workflows/test-deploy-universal.yml index f7479b2eef..c3205e13f3 100644 --- a/.github/workflows/test-deploy-universal.yml +++ b/.github/workflows/test-deploy-universal.yml @@ -77,16 +77,23 @@ jobs: - name: Free disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 with: - tool-cache: false + tool-cache: true android: true dotnet: true haskell: true - large-packages: false - docker-images: false - swap-storage: false + large-packages: true + docker-images: true + swap-storage: true - uses: actions/checkout@v6 + # Replace the default ~4 GB runner swap with a larger file so transient + # memory spikes (e.g. peertube plugin install, issue #162) don't trip + # the host OOM-killer. Script picks / or /mnt based on live free space. + - name: Enlarge swap space + shell: bash + run: ./scripts/github/enlarge_swap.sh + - name: Resolve repository owner shell: bash run: | diff --git a/.github/workflows/test-deploy-workstation.yml b/.github/workflows/test-deploy-workstation.yml index 26e538773a..8f53489e91 100644 --- a/.github/workflows/test-deploy-workstation.yml +++ b/.github/workflows/test-deploy-workstation.yml @@ -77,16 +77,23 @@ jobs: - name: Free disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 with: - tool-cache: false + tool-cache: true android: true dotnet: true haskell: true - large-packages: false - docker-images: false - swap-storage: false + large-packages: true + docker-images: true + swap-storage: true - uses: actions/checkout@v6 + # Replace the default ~4 GB runner swap with a larger file so transient + # memory spikes (e.g. peertube plugin install, issue #162) don't trip + # the host OOM-killer. Script picks / or /mnt based on live free space. + - name: Enlarge swap space + shell: bash + run: ./scripts/github/enlarge_swap.sh + - name: Resolve repository owner shell: bash run: | diff --git a/docs/contributing/artefact/files/github/workflow.md b/docs/contributing/artefact/files/github/workflow.md index a35fce7972..430ac8425f 100644 --- a/docs/contributing/artefact/files/github/workflow.md +++ b/docs/contributing/artefact/files/github/workflow.md @@ -50,20 +50,47 @@ name: "๐Ÿšซ Cancel: PR Runs on Close" ## Disk space ๐Ÿ’พ -Deploy test workflows use the `jlumbroso/free-disk-space` action to reclaim runner space before Docker pulls start. +Deploy test workflows use the `jlumbroso/free-disk-space` action to reclaim runner space before Docker pulls start. Because the actual deploy runs **inside** the `infinito` container, the host's language toolchains, build packages, and default Docker layer cache are all dead weight and MUST be reclaimed aggressively. | Option | Value | Reason | |---|---|---| +| `tool-cache` | `true` | Deploy runs inside the `infinito` container; host Node/Python/Go/Ruby toolchains go unused | | `android` | `true` | Android SDK is never needed; safe to remove | | `dotnet` | `true` | .NET SDK is never needed; safe to remove | | `haskell` | `true` | Haskell toolchain is never needed; safe to remove | -| `large-packages` | `false` | Can remove build tools (gcc, Python headers) required by pip/Ansible | -| `docker-images` | `false` | Can remove cached layers reused between deploy steps | -| `swap-storage` | `false` | Disabling swap can cause OOM for memory-intensive services (e.g. Keycloak) | -| `tool-cache` | `false` | Runner tool cache may be needed by subsequent steps | +| `large-packages` | `true` | Ansible, pip and gcc run **inside** the container, not on the host | +| `docker-images` | `true` | Matrix jobs don't share a Docker layer cache; each runner pulls its own `infinito` image | +| `swap-storage` | `true` | Default swap file is replaced by `pierotofy/set-swap-space` (see below) | -Only options that are **guaranteed unused** MUST be set to `true`. -When in doubt, keep the option at `false`. +Set an option back to `false` only when a new host-side step in the same workflow genuinely needs the removed payload. + +## Swap ๐Ÿ’พ + +Deploy test workflows enlarge host swap via [enlarge_swap.sh](../../../../../scripts/github/enlarge_swap.sh) to absorb transient memory spikes (e.g. PeerTube plugin install [#162](https://github.com/infinito-nexus/core/issues/162)) that would otherwise trip the host OOM-killer on the 16 GB GitHub-hosted runner. + +The script **prefers `/`** for deterministic placement and falls back to `/mnt` only when `/` does not have enough headroom. Preferring `/` avoids surprises from runners where `/mnt` is unexpectedly pre-populated (larger-runner images, custom Docker-data-root relocations, self-hosted setups). + +Swap size is computed dynamically: `free(/) - buffer`. The buffer reserves space for nested Docker layers, the checkout, and the runner cache. With the aggressive `free-disk-space` options above, a ~25 GB buffer typically leaves a 25โ€“30 GB swapfile on `ubuntu-latest`, which is far more than any workload the deploy jobs currently trigger. + +| Argument | Default | Reason | +|---|---|---| +| `buffer-gb` (positional) | `25` | Reserves headroom for nested Docker data, checkout and runtime state; pass a smaller value only when you know the job footprint shrinks | + +Workflow step invocation: + +```yaml +- name: Enlarge swap space + shell: bash + run: ./scripts/github/enlarge_swap.sh +``` + +Ordering: + +- Swap step MUST run **after** `actions/checkout` because the script lives inside the repo. +- Swap step SHOULD run **after** `jlumbroso/free-disk-space` so the reclaimed disk on `/` is also a candidate target when `/mnt` is crowded. +- Swap step MUST run **before** any `make up` / container build step so the expanded swap is active when heavy-allocation work begins. + +Swap is a host-kernel resource; see [svc-opt-swapfile](../../../../../roles/svc-opt-swapfile/) for why the in-stack swap role is intentionally skipped inside containers. ## Separation of concerns ๐Ÿงฉ diff --git a/scripts/github/enlarge_swap.sh b/scripts/github/enlarge_swap.sh new file mode 100755 index 0000000000..2397d99fbf --- /dev/null +++ b/scripts/github/enlarge_swap.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Enlarge host swap on a GitHub-hosted runner. Consumes all remaining +# space on / minus a reserved buffer for Docker layers, checkout and +# runtime data. Falls back to /mnt when / is too small. Replaces +# whatever swap was already active. Needed on 16 GB runners where +# transient spikes (e.g. peertube plugin install, issue #162) otherwise +# trip the host OOM-killer. +# +# Usage: enlarge_swap.sh [buffer-gb] +# Default buffer: 25 GB (nested Docker images + checkout + runner cache). + +set -euo pipefail + +BUFFER_GB="${1:-25}" +MIN_SWAP_GB=4 # refuse to create a swapfile smaller than this + +sudo swapoff -a || true +sudo rm -f /mnt/swapfile /swapfile + +root_free=$(df --output=avail -B1G / | tail -n1 | tr -d ' ') +mnt_free=0 +if [ -d /mnt ]; then + mnt_free=$(df --output=avail -B1G /mnt | tail -n1 | tr -d ' ') +fi + +root_swap=$((root_free - BUFFER_GB)) +mnt_swap=$((mnt_free - BUFFER_GB)) + +if [ "$root_swap" -ge "$MIN_SWAP_GB" ]; then + target=/swapfile + size=$root_swap +elif [ "$mnt_swap" -ge "$MIN_SWAP_GB" ]; then + target=/mnt/swapfile + size=$mnt_swap +else + echo "Neither / nor /mnt has ${MIN_SWAP_GB}+${BUFFER_GB} GB free for swap" >&2 + df -h / /mnt >&2 + exit 1 +fi + +sudo fallocate -l "${size}G" "$target" +sudo chmod 600 "$target" +sudo mkswap "$target" +sudo swapon "$target" + +echo "Swap of ${size}G created at ${target} (buffer ${BUFFER_GB}G)" +free -h +swapon --show From 0cfe0b4d512dce01e0db784effddc6a4840c035b Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 03:23:37 +0200 Subject: [PATCH 07/13] Normalize whitespace in enlarge_swap.sh Co-Authored-By: Claude Opus 4.7 --- scripts/github/enlarge_swap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/github/enlarge_swap.sh b/scripts/github/enlarge_swap.sh index 2397d99fbf..60cb00c59c 100755 --- a/scripts/github/enlarge_swap.sh +++ b/scripts/github/enlarge_swap.sh @@ -12,7 +12,7 @@ set -euo pipefail BUFFER_GB="${1:-25}" -MIN_SWAP_GB=4 # refuse to create a swapfile smaller than this +MIN_SWAP_GB=4 # refuse to create a swapfile smaller than this sudo swapoff -a || true sudo rm -f /mnt/swapfile /swapfile From d0c9f9dd93d2b19e96add5d5b219e4f72ecd86bd Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 03:31:19 +0200 Subject: [PATCH 08/13] Mark mailu internal sub-services as shared=false The 11 mailu sub-containers (admin, imap, smtp, antispam, antivirus, oletools, webdav, fetchmail, webmail, resolver, front) are internal to mailu's multi-container decomposition, not shared with other roles. Add explicit shared=false so the service shared/enabled consistency lint passes. Co-Authored-By: Claude Opus 4.7 --- roles/web-app-mailu/config/main.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/roles/web-app-mailu/config/main.yml b/roles/web-app-mailu/config/main.yml index 330fc95ca5..e21a7f09cc 100644 --- a/roles/web-app-mailu/config/main.yml +++ b/roles/web-app-mailu/config/main.yml @@ -62,66 +62,77 @@ compose: pids_limit: 512 admin: enabled: true + shared: false cpus: "0.3" mem_reservation: "512m" mem_limit: "1g" pids_limit: 512 imap: enabled: true + shared: false cpus: "0.5" mem_reservation: "1g" mem_limit: "1.5g" pids_limit: 512 smtp: enabled: true + shared: false cpus: "0.5" mem_reservation: "1g" mem_limit: "1.5g" pids_limit: 512 antispam: enabled: true + shared: false cpus: "0.6" mem_reservation: "1g" mem_limit: "1.5g" pids_limit: 512 antivirus: enabled: true + shared: false cpus: "0.6" mem_reservation: "2g" mem_limit: "3g" pids_limit: 512 oletools: enabled: true + shared: false cpus: "0.2" mem_reservation: "256m" mem_limit: "512m" pids_limit: 256 webdav: enabled: true + shared: false cpus: "0.2" mem_reservation: "256m" mem_limit: "512m" pids_limit: 256 fetchmail: enabled: true + shared: false cpus: "0.2" mem_reservation: "256m" mem_limit: "512m" pids_limit: 256 webmail: enabled: true + shared: false cpus: "0.3" mem_reservation: "512m" mem_limit: "1g" pids_limit: 512 resolver: enabled: true + shared: false cpus: "0.2" mem_reservation: "256m" mem_limit: "512m" pids_limit: 256 front: enabled: true + shared: false cpus: "0.3" mem_reservation: "512m" mem_limit: "1g" From 6fac8996ffa4279728b0d02f9b19628f15e8c69d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 12:37:54 +0200 Subject: [PATCH 09/13] Add runner-state diagnostic helper for CI deploy debugging Extracts disk/memory/docker/swap dump into a reusable helper, calls it pre/post swap resize, and runs it on deploy failure across all four deploy workflows. Also adds SKIP_SWAP=1 short-circuit for A/B probing swap-related regressions. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/test-deploy-local.yml | 5 ++ .github/workflows/test-deploy-server.yml | 5 ++ .github/workflows/test-deploy-universal.yml | 5 ++ .github/workflows/test-deploy-workstation.yml | 5 ++ scripts/github/dump_runner_state.sh | 61 +++++++++++++++++++ scripts/github/enlarge_swap.sh | 24 +++++++- 6 files changed, 102 insertions(+), 3 deletions(-) create mode 100755 scripts/github/dump_runner_state.sh diff --git a/.github/workflows/test-deploy-local.yml b/.github/workflows/test-deploy-local.yml index 0f5df083d0..37a9ead21a 100644 --- a/.github/workflows/test-deploy-local.yml +++ b/.github/workflows/test-deploy-local.yml @@ -122,3 +122,8 @@ jobs: DISTROS: ${{ needs.discover.outputs.distros }} APPS: ${{ matrix.apps }} run: ./scripts/tests/deploy/ci/all.sh + + - name: Dump runner state (on failure) + if: failure() + shell: bash + run: ./scripts/github/dump_runner_state.sh "post-deploy failure (${{ matrix.apps }})" diff --git a/.github/workflows/test-deploy-server.yml b/.github/workflows/test-deploy-server.yml index e7ae1409ed..6a61530cf4 100644 --- a/.github/workflows/test-deploy-server.yml +++ b/.github/workflows/test-deploy-server.yml @@ -115,6 +115,11 @@ jobs: APPS: ${{ matrix.apps }} run: ./scripts/tests/deploy/ci/all.sh + - name: Dump runner state (on failure) + if: failure() + shell: bash + run: ./scripts/github/dump_runner_state.sh "post-deploy failure (${{ matrix.apps }})" + - name: Upload Playwright artifacts if: always() uses: actions/upload-artifact@v7 diff --git a/.github/workflows/test-deploy-universal.yml b/.github/workflows/test-deploy-universal.yml index c3205e13f3..1f6e96ac7e 100644 --- a/.github/workflows/test-deploy-universal.yml +++ b/.github/workflows/test-deploy-universal.yml @@ -109,3 +109,8 @@ jobs: env: APPS: ${{ matrix.apps }} run: ./scripts/tests/deploy/ci/all.sh + + - name: Dump runner state (on failure) + if: failure() + shell: bash + run: ./scripts/github/dump_runner_state.sh "post-deploy failure (${{ matrix.apps }})" diff --git a/.github/workflows/test-deploy-workstation.yml b/.github/workflows/test-deploy-workstation.yml index 8f53489e91..3716110a1e 100644 --- a/.github/workflows/test-deploy-workstation.yml +++ b/.github/workflows/test-deploy-workstation.yml @@ -109,3 +109,8 @@ jobs: env: APPS: ${{ matrix.apps }} run: ./scripts/tests/deploy/ci/all.sh + + - name: Dump runner state (on failure) + if: failure() + shell: bash + run: ./scripts/github/dump_runner_state.sh "post-deploy failure (${{ matrix.apps }})" diff --git a/scripts/github/dump_runner_state.sh b/scripts/github/dump_runner_state.sh new file mode 100755 index 0000000000..fadf990aa3 --- /dev/null +++ b/scripts/github/dump_runner_state.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Dump disk, memory, swap and docker state of a GitHub-hosted runner. +# Intended as a diagnostic helper that can be called from anywhere +# inside a workflow step (pre/post swap resize, post-deploy failure). +# +# Usage: dump_runner_state.sh [label] +# The label is rendered into the log group title and the step summary. + +set -euo pipefail + +LABEL="${1:-runner state}" + +echo "::group::Runner state โ€” ${LABEL}" + +echo "--- date ---" +date -u +'%Y-%m-%dT%H:%M:%SZ' + +echo "--- disk (bytes) ---" +df -h / /mnt 2>/dev/null || df -h / + +echo "--- disk (inodes) ---" +df -hi / /mnt 2>/dev/null || df -hi / + +echo "--- block devices ---" +lsblk -f || true + +echo "--- mounts (/, /mnt, /var/lib/docker, /mnt/docker) ---" +mount | grep -E ' / | /mnt | /var/lib/docker | /mnt/docker ' || true + +echo "--- docker ---" +docker info 2>/dev/null | + grep -E 'Docker Root Dir|Storage Driver|Containers|Images' || + echo "docker daemon not reachable" + +echo "--- memory ---" +free -h + +echo "--- swap ---" +swapon --show || echo "no swap active" + +echo "--- largest dirs on / (top 15) ---" +sudo du -xh --max-depth=2 / 2>/dev/null | sort -h | tail -15 || true + +if [ -d /mnt ]; then + echo "--- largest dirs on /mnt (top 15) ---" + sudo du -xh --max-depth=2 /mnt 2>/dev/null | sort -h | tail -15 || true +fi + +echo "::endgroup::" + +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + root_free=$(df --output=avail -B1G / 2>/dev/null | tail -n1 | tr -d ' ' || echo '?') + mnt_free=$(df --output=avail -B1G /mnt 2>/dev/null | tail -n1 | tr -d ' ' || echo '?') + mem_free=$(free -g | awk '/^Mem:/ {print $4"G"}') + swap_total=$(free -g | awk '/^Swap:/ {print $2"G"}') + { + echo "### Runner state โ€” ${LABEL}" + echo "- / free: **${root_free}G** ยท /mnt free: **${mnt_free}G**" + echo "- mem free: **${mem_free}** ยท swap total: **${swap_total}**" + } >>"$GITHUB_STEP_SUMMARY" +fi diff --git a/scripts/github/enlarge_swap.sh b/scripts/github/enlarge_swap.sh index 60cb00c59c..4466289605 100755 --- a/scripts/github/enlarge_swap.sh +++ b/scripts/github/enlarge_swap.sh @@ -8,12 +8,26 @@ # # Usage: enlarge_swap.sh [buffer-gb] # Default buffer: 25 GB (nested Docker images + checkout + runner cache). +# +# Debug toggle: set SKIP_SWAP=1 to leave the runner's default swap +# untouched (A/B probe for issues caused by this script itself). set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DUMP="${HERE}/dump_runner_state.sh" + +if [ "${SKIP_SWAP:-0}" = "1" ]; then + echo "SKIP_SWAP=1 โ€” leaving default runner swap in place." + "$DUMP" "skip-swap (SKIP_SWAP=1)" + exit 0 +fi + BUFFER_GB="${1:-25}" MIN_SWAP_GB=4 # refuse to create a swapfile smaller than this +"$DUMP" "before swap resize" + sudo swapoff -a || true sudo rm -f /mnt/swapfile /swapfile @@ -26,23 +40,27 @@ fi root_swap=$((root_free - BUFFER_GB)) mnt_swap=$((mnt_free - BUFFER_GB)) +echo "Candidates: / has ${root_free}G free โ†’ up to ${root_swap}G swap; /mnt has ${mnt_free}G free โ†’ up to ${mnt_swap}G swap (buffer ${BUFFER_GB}G)" + if [ "$root_swap" -ge "$MIN_SWAP_GB" ]; then target=/swapfile size=$root_swap + chosen="/ (preferred)" elif [ "$mnt_swap" -ge "$MIN_SWAP_GB" ]; then target=/mnt/swapfile size=$mnt_swap + chosen="/mnt (fallback)" else echo "Neither / nor /mnt has ${MIN_SWAP_GB}+${BUFFER_GB} GB free for swap" >&2 df -h / /mnt >&2 exit 1 fi +echo "Chose ${chosen}: creating ${size}G swapfile at ${target}" + sudo fallocate -l "${size}G" "$target" sudo chmod 600 "$target" sudo mkswap "$target" sudo swapon "$target" -echo "Swap of ${size}G created at ${target} (buffer ${BUFFER_GB}G)" -free -h -swapon --show +"$DUMP" "after swap resize โ†’ ${size}G at ${target}" From 033b045cf04fdb75f91263e28a907124531fd85d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 12:44:37 +0200 Subject: [PATCH 10/13] Add nested-container diagnostics to runner-state dump Surfaces df, failed units, and hlth-disc-space status/journal from inside the infinito container so failure-time dumps reflect the same filesystem view that the disk-space health check evaluates, not just the host. Co-Authored-By: Claude Opus 4.7 --- scripts/github/dump_runner_state.sh | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/scripts/github/dump_runner_state.sh b/scripts/github/dump_runner_state.sh index fadf990aa3..9386728909 100755 --- a/scripts/github/dump_runner_state.sh +++ b/scripts/github/dump_runner_state.sh @@ -46,6 +46,37 @@ if [ -d /mnt ]; then sudo du -xh --max-depth=2 /mnt 2>/dev/null | sort -h | tail -15 || true fi +# Nested view โ€” hlth-disc-space runs INSIDE the infinito container and +# evaluates df there, so host-side df alone can be misleading. These +# blocks silently no-op when the container is not (yet) running. +echo "--- infinito container (nested view) ---" +INFINITO_CONTAINER="$(docker ps --filter 'name=infinito_nexus' --format '{{.Names}}' 2>/dev/null | head -1 || true)" +if [ -n "${INFINITO_CONTAINER}" ]; then + echo "container: ${INFINITO_CONTAINER}" + + echo "--- df inside container ---" + docker exec "${INFINITO_CONTAINER}" df -h 2>&1 || echo "docker exec df failed" + + echo "--- df --output=pcent inside container (what hlth-disc-space sees) ---" + docker exec "${INFINITO_CONTAINER}" df --output=pcent 2>&1 || true + + echo "--- failed systemd units inside container ---" + docker exec "${INFINITO_CONTAINER}" systemctl --failed --no-pager 2>&1 || + echo "systemctl --failed not reachable" + + echo "--- hlth-disc-space status ---" + docker exec "${INFINITO_CONTAINER}" \ + systemctl status 'hlth-disc-space*' --no-pager -l 2>&1 || + echo "no hlth-disc-space unit (not deployed yet)" + + echo "--- hlth-disc-space journal (last 80 lines) ---" + docker exec "${INFINITO_CONTAINER}" \ + journalctl -u 'hlth-disc-space*' -n 80 --no-pager 2>&1 || + echo "journal for hlth-disc-space not accessible" +else + echo "no infinito container running (expected before 'make up')" +fi + echo "::endgroup::" if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then From fe18ae43cfcd417c34ac60d367606ef247537f85 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 16:54:03 +0200 Subject: [PATCH 11/13] Fix enlarge_swap.sh starving deploys of disk space (#162) Swap size was free(/)-25, which took ~99 GB on the 145 GB root fs. Remaining ~25 GB then ran out mid-deploy during Playwright image pulls (ENOSPC). Replace with a fixed 16 GB cap and use mountpoint to detect a real /mnt partition instead of [ -d /mnt ], which matched the empty placeholder dir on single-partition runners. Co-Authored-By: Claude Opus 4.7 --- .../artefact/files/github/workflow.md | 6 +- scripts/github/enlarge_swap.sh | 69 ++++++++++--------- 2 files changed, 40 insertions(+), 35 deletions(-) diff --git a/docs/contributing/artefact/files/github/workflow.md b/docs/contributing/artefact/files/github/workflow.md index 430ac8425f..807a9d09f9 100644 --- a/docs/contributing/artefact/files/github/workflow.md +++ b/docs/contributing/artefact/files/github/workflow.md @@ -68,13 +68,13 @@ Set an option back to `false` only when a new host-side step in the same workflo Deploy test workflows enlarge host swap via [enlarge_swap.sh](../../../../../scripts/github/enlarge_swap.sh) to absorb transient memory spikes (e.g. PeerTube plugin install [#162](https://github.com/infinito-nexus/core/issues/162)) that would otherwise trip the host OOM-killer on the 16 GB GitHub-hosted runner. -The script **prefers `/`** for deterministic placement and falls back to `/mnt` only when `/` does not have enough headroom. Preferring `/` avoids surprises from runners where `/mnt` is unexpectedly pre-populated (larger-runner images, custom Docker-data-root relocations, self-hosted setups). +The script **prefers `/mnt`** when it is a separate partition (classic `ubuntu-latest` layout with `/dev/sdb` mounted at `/mnt`) and falls back to `/` otherwise. Current public runners no longer expose a separate `/mnt`, so the swapfile lands on `/` and must not starve the root filesystem. -Swap size is computed dynamically: `free(/) - buffer`. The buffer reserves space for nested Docker layers, the checkout, and the runner cache. With the aggressive `free-disk-space` options above, a ~25 GB buffer typically leaves a 25โ€“30 GB swapfile on `ubuntu-latest`, which is far more than any workload the deploy jobs currently trigger. +Swap size is a **fixed constant**, not a "free space minus buffer" calculation. An earlier version took ~99 GB on a 145 GB root partition, leaving so little headroom that later Docker layer writes (Playwright test image pull) failed with `no space left on device`. A fixed cap is large enough to absorb the peertube spike and small enough that Docker layer writes keep succeeding throughout the 30+ min deploy. | Argument | Default | Reason | |---|---|---| -| `buffer-gb` (positional) | `25` | Reserves headroom for nested Docker data, checkout and runtime state; pass a smaller value only when you know the job footprint shrinks | +| `size-gb` (positional) | `16` | Empirically sufficient for the peertube memory spike ([#162](https://github.com/infinito-nexus/core/issues/162)); raise only when a workload is proven to need more | Workflow step invocation: diff --git a/scripts/github/enlarge_swap.sh b/scripts/github/enlarge_swap.sh index 4466289605..945266437c 100755 --- a/scripts/github/enlarge_swap.sh +++ b/scripts/github/enlarge_swap.sh @@ -1,13 +1,19 @@ #!/usr/bin/env bash -# Enlarge host swap on a GitHub-hosted runner. Consumes all remaining -# space on / minus a reserved buffer for Docker layers, checkout and -# runtime data. Falls back to /mnt when / is too small. Replaces -# whatever swap was already active. Needed on 16 GB runners where -# transient spikes (e.g. peertube plugin install, issue #162) otherwise -# trip the host OOM-killer. +# Enlarge host swap on a GitHub-hosted runner. Replaces the runner's +# default ~4 GB swap with a fixed-size swapfile so transient memory +# spikes (e.g. peertube plugin install, issue #162) don't trip the host +# OOM-killer, WITHOUT starving the rest of the job of disk space. # -# Usage: enlarge_swap.sh [buffer-gb] -# Default buffer: 25 GB (nested Docker images + checkout + runner cache). +# Prefers /mnt when it is a separate partition (classic ubuntu-latest +# layout with /dev/sdb mounted at /mnt). On current public runners +# there is no separate /mnt partition โ€” everything lives on /, so we +# place the swapfile there. The swap size is a hard constant, never a +# "free minus buffer" computation: a previous version consumed ~99 GB +# on a 145 GB root, leaving so little headroom that later Docker layer +# writes (Playwright test image pull) failed with ENOSPC. See #162. +# +# Usage: enlarge_swap.sh [size-gb] +# Default: 16 GB (empirically sufficient for the peertube spike). # # Debug toggle: set SKIP_SWAP=1 to leave the runner's default swap # untouched (A/B probe for issues caused by this script itself). @@ -23,44 +29,43 @@ if [ "${SKIP_SWAP:-0}" = "1" ]; then exit 0 fi -BUFFER_GB="${1:-25}" -MIN_SWAP_GB=4 # refuse to create a swapfile smaller than this +SWAP_GB="${1:-16}" +MIN_FREE_GB=$((SWAP_GB + 4)) # need the target size plus a little slack "$DUMP" "before swap resize" sudo swapoff -a || true sudo rm -f /mnt/swapfile /swapfile -root_free=$(df --output=avail -B1G / | tail -n1 | tr -d ' ') -mnt_free=0 -if [ -d /mnt ]; then +# Only treat /mnt as a candidate if it is actually a separate mount. +# On current ubuntu-latest public runners /mnt is an empty directory +# on the root fs, which would double-count free space. +if mountpoint -q /mnt 2>/dev/null; then mnt_free=$(df --output=avail -B1G /mnt | tail -n1 | tr -d ' ') -fi - -root_swap=$((root_free - BUFFER_GB)) -mnt_swap=$((mnt_free - BUFFER_GB)) - -echo "Candidates: / has ${root_free}G free โ†’ up to ${root_swap}G swap; /mnt has ${mnt_free}G free โ†’ up to ${mnt_swap}G swap (buffer ${BUFFER_GB}G)" - -if [ "$root_swap" -ge "$MIN_SWAP_GB" ]; then - target=/swapfile - size=$root_swap - chosen="/ (preferred)" -elif [ "$mnt_swap" -ge "$MIN_SWAP_GB" ]; then - target=/mnt/swapfile - size=$mnt_swap - chosen="/mnt (fallback)" else - echo "Neither / nor /mnt has ${MIN_SWAP_GB}+${BUFFER_GB} GB free for swap" >&2 + mnt_free=0 +fi +root_free=$(df --output=avail -B1G / | tail -n1 | tr -d ' ') + +echo "Target swap: ${SWAP_GB}G. Free: / ${root_free}G, /mnt ${mnt_free}G (separate mount: $(mountpoint -q /mnt 2>/dev/null && echo yes || echo no))" + +if [ "$mnt_free" -ge "$MIN_FREE_GB" ]; then + target=/mnt/swapfile + chosen="/mnt (separate partition)" +elif [ "$root_free" -ge "$MIN_FREE_GB" ]; then + target=/swapfile + chosen="/ (no separate /mnt)" +else + echo "Neither / nor /mnt has ${MIN_FREE_GB} GB free for a ${SWAP_GB} GB swapfile" >&2 df -h / /mnt >&2 exit 1 fi -echo "Chose ${chosen}: creating ${size}G swapfile at ${target}" +echo "Chose ${chosen}: creating ${SWAP_GB}G swapfile at ${target}" -sudo fallocate -l "${size}G" "$target" +sudo fallocate -l "${SWAP_GB}G" "$target" sudo chmod 600 "$target" sudo mkswap "$target" sudo swapon "$target" -"$DUMP" "after swap resize โ†’ ${size}G at ${target}" +"$DUMP" "after swap resize โ†’ ${SWAP_GB}G at ${target}" From 6b5e4356a64da77478d056e65969411224d4ef5b Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 22:05:45 +0200 Subject: [PATCH 12/13] Add peertube plugin-install diagnostics on failure Wraps the auth-openid-connect plugin install in block/rescue so that when the container exec trips (e.g. "Container is restarting", cgroup OOM kill), a dedicated script dumps container state, health, resource limits, stats, last 250 log lines, host memory+swap, and dmesg OOM-killer evidence before the task fails explicitly. Makes the actual cause visible in CI without another round-trip. Co-Authored-By: Claude Opus 4.7 --- .../files/dump_peertube_diagnostics.sh | 67 +++++++++++++++++++ .../web-app-peertube/tasks/01_enable-oidc.yml | 52 +++++++++----- 2 files changed, 103 insertions(+), 16 deletions(-) create mode 100755 roles/web-app-peertube/files/dump_peertube_diagnostics.sh diff --git a/roles/web-app-peertube/files/dump_peertube_diagnostics.sh b/roles/web-app-peertube/files/dump_peertube_diagnostics.sh new file mode 100755 index 0000000000..b038bdccf0 --- /dev/null +++ b/roles/web-app-peertube/files/dump_peertube_diagnostics.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# roles/web-app-peertube/files/dump_peertube_diagnostics.sh +# +# Best-effort diagnostics for the PeerTube container after a plugin-install +# failure. Prints container state, health, resource limits, logs, stats, +# host memory, and OOM-killer evidence. Always exits 0 โ€” the ansible rescue +# step is responsible for re-raising the failure after the dump. +# +# Usage: dump_peertube_diagnostics.sh +# Env: PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB, PEERTUBE_MAX_OLD_SPACE_SIZE +# (optional โ€” printed into the report for correlation). + +set -u + +CONTAINER="${1:-peertube}" +INSTALL_HEAP_MB="${PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB:-?}" +MAIN_HEAP_MB="${PEERTUBE_MAX_OLD_SPACE_SIZE:-?}" + +echo "============================================================" +echo "=== PEERTUBE DIAGNOSTICS (plugin-install failure)" +echo "============================================================" +echo "container=${CONTAINER}" +echo "install_heap_mb=${INSTALL_HEAP_MB} main_heap_mb=${MAIN_HEAP_MB}" +echo + +echo "--- State ---" +container inspect --format \ + 'Status={{.State.Status}} Running={{.State.Running}} Restarting={{.State.Restarting}} OOMKilled={{.State.OOMKilled}} ExitCode={{.State.ExitCode}} StartedAt={{.State.StartedAt}} FinishedAt={{.State.FinishedAt}} RestartCount={{.RestartCount}}' \ + "${CONTAINER}" 2>&1 || true +echo + +echo "--- Health ---" +container inspect --format \ + '{{if .State.Health}}Status={{.State.Health.Status}} FailingStreak={{.State.Health.FailingStreak}}{{range .State.Health.Log}}{{println "log:" .ExitCode .Output}}{{end}}{{else}}no healthcheck configured{{end}}' \ + "${CONTAINER}" 2>&1 || true +echo + +echo "--- Resource limits ---" +container inspect --format \ + 'MemLimit={{.HostConfig.Memory}} MemSwap={{.HostConfig.MemorySwap}} NanoCpus={{.HostConfig.NanoCpus}} PidsLimit={{.HostConfig.PidsLimit}}' \ + "${CONTAINER}" 2>&1 || true +echo + +echo "--- container stats (single snapshot) ---" +container stats --no-stream --no-trunc "${CONTAINER}" 2>&1 || true +echo + +echo "--- container logs (last 250 lines) ---" +container logs --tail 250 "${CONTAINER}" 2>&1 || true +echo + +echo "--- Host memory + swap ---" +free -h 2>&1 || true +echo +swapon --show 2>&1 || true +echo + +echo "--- dmesg tail (OOM-killer evidence, best effort) ---" +dmesg -T 2>/dev/null | tail -n 80 | grep -Ei 'oom|kill|memory' \ + || echo 'no oom/kill evidence in dmesg tail' +echo + +echo "============================================================" +echo "=== END PEERTUBE DIAGNOSTICS" +echo "============================================================" + +exit 0 diff --git a/roles/web-app-peertube/tasks/01_enable-oidc.yml b/roles/web-app-peertube/tasks/01_enable-oidc.yml index 5006758e20..5d18eea01c 100644 --- a/roles/web-app-peertube/tasks/01_enable-oidc.yml +++ b/roles/web-app-peertube/tasks/01_enable-oidc.yml @@ -10,23 +10,43 @@ failed_when: false changed_when: false -- name: "Install auth-openid-connect plugin for Peertube" - # The plugin install spawns a fresh node process under the running PeerTube - # container. Without an explicit heap cap, V8 inherits the container-wide - # NODE_OPTIONS (~35% of mem_limit, multi-GB) and can push a constrained CI - # runner over its host RAM, getting SIGKILLed (rc=137) before the install - # completes. Force a tight heap and retry to absorb transient host pressure. - command: > - container exec - -e NODE_OPTIONS=--max-old-space-size={{ PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB }} - {{ PEERTUBE_CONTAINER }} npm run plugin:install -- --npm-name {{ PEERTUBE_OIDC_PLUGIN }} - register: peertube_oidc_install - changed_when: peertube_oidc_install.rc == 0 - retries: 3 - delay: 15 - until: peertube_oidc_install.rc == 0 +- name: "Install auth-openid-connect plugin for Peertube (with diagnostics on failure)" when: peertube_oidc_plugin_check.rc != 0 - notify: compose up + block: + + - name: "Install auth-openid-connect plugin for Peertube" + # The plugin install spawns a fresh node process under the running PeerTube + # container. Without an explicit heap cap, V8 inherits the container-wide + # NODE_OPTIONS (~35% of mem_limit, multi-GB) and can push a constrained CI + # runner over its host RAM, getting SIGKILLed (rc=137) before the install + # completes. Force a tight heap and retry to absorb transient host pressure. + ansible.builtin.command: > + container exec + -e NODE_OPTIONS=--max-old-space-size={{ PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB }} + {{ PEERTUBE_CONTAINER }} npm run plugin:install -- --npm-name {{ PEERTUBE_OIDC_PLUGIN }} + register: peertube_oidc_install + changed_when: peertube_oidc_install.rc == 0 + retries: 3 + delay: 15 + until: peertube_oidc_install.rc == 0 + notify: compose up + + rescue: + + - name: "PEERTUBE | Dump diagnostics after install failure" + ansible.builtin.script: "{{ role_path }}/files/dump_peertube_diagnostics.sh {{ PEERTUBE_CONTAINER }}" + environment: + PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB: "{{ PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB }}" + PEERTUBE_MAX_OLD_SPACE_SIZE: "{{ PEERTUBE_MAX_OLD_SPACE_SIZE }}" + changed_when: false + failed_when: false + + - name: "PEERTUBE | Fail explicitly after diagnostics" + ansible.builtin.fail: + msg: >- + Installing peertube-plugin-auth-openid-connect failed. + See PEERTUBE DIAGNOSTICS block above for state, OOMKilled, exit + code, logs, stats, host memory, and dmesg OOM-killer evidence. - name: "Update the settings column of the auth-openid-connect plugin" community.postgresql.postgresql_query: From 97ee7e65ea448f65616831c7cec8c988aaadb5e7 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Mon, 20 Apr 2026 22:55:08 +0200 Subject: [PATCH 13/13] Bump peertube cgroup mem_limit during plugin install Plugin install via `container exec` shares the cgroup with the long-running peertube node process. Combined RSS was tripping the 8g mem_limit and SIGKILLing peertube, which surfaced as "Container is restarting" on subsequent exec attempts. - Temp raise mem_limit + unlimited swap for the install window, restore the compose-defined value on the happy path only - Derive the bumped ceiling from PEERTUBE_MEM_LIMIT + install heap cap via a new role-local `install_mem_limit` filter (overhead=4 covers native RSS and spawned npm/node-gyp workers) - Extract the SQL upsert into templates/upsert-oidc-plugin.sql.j2 - Move enable/disable OIDC tasks under tasks/oidc/ and rename the diagnostics script - Document the extract-code convention in docs/contributing/tools/ansible.md Co-Authored-By: Claude Opus 4.7 --- docs/contributing/tools/ansible.md | 10 ++ ...peertube_diagnostics.sh => diagnostics.sh} | 4 +- .../filter_plugins/install_mem_limit.py | 77 ++++++++++++ roles/web-app-peertube/tasks/main.yml | 4 +- .../{02_disable-oidc.yml => oidc/disable.yml} | 0 .../{01_enable-oidc.yml => oidc/enable.yml} | 55 ++++++--- .../templates/upsert-oidc-plugin.sql.j2 | 10 ++ roles/web-app-peertube/vars/main.yml | 14 +++ tests/unit/roles/web-app-peertube/__init__.py | 0 .../filter_plugins/__init__.py | 0 .../filter_plugins/test_install_mem_limit.py | 112 ++++++++++++++++++ 11 files changed, 264 insertions(+), 22 deletions(-) rename roles/web-app-peertube/files/{dump_peertube_diagnostics.sh => diagnostics.sh} (95%) create mode 100644 roles/web-app-peertube/filter_plugins/install_mem_limit.py rename roles/web-app-peertube/tasks/{02_disable-oidc.yml => oidc/disable.yml} (100%) rename roles/web-app-peertube/tasks/{01_enable-oidc.yml => oidc/enable.yml} (54%) create mode 100644 roles/web-app-peertube/templates/upsert-oidc-plugin.sql.j2 create mode 100644 tests/unit/roles/web-app-peertube/__init__.py create mode 100644 tests/unit/roles/web-app-peertube/filter_plugins/__init__.py create mode 100644 tests/unit/roles/web-app-peertube/filter_plugins/test_install_mem_limit.py diff --git a/docs/contributing/tools/ansible.md b/docs/contributing/tools/ansible.md index 54637d95b4..0eb936d9da 100644 --- a/docs/contributing/tools/ansible.md +++ b/docs/contributing/tools/ansible.md @@ -35,6 +35,16 @@ Use this rule when a role pins container image versions in `roles/web-*/config/m - You MUST use `# nocheck: docker-version` directly above a `version:` key only for intentional exceptions that should not be flagged by the external image-version check. - You SHOULD prefer explicit version pinning over drifting tags when the role depends on stable, reproducible deploys. +### Extracted Code ๐Ÿ“ค + +Use this rule when a task body contains non-trivial shell, SQL, Python, JavaScript, or any other non-YAML code. + +- You MUST extract non-trivial non-YAML code out of task files into a dedicated role asset instead of inlining it: + - Static scripts (shell, Python, binaries) belong in `roles//files/.` and are invoked via `ansible.builtin.script` or `command`. + - Rendered code (SQL, JSON, JS, config snippets, env files) belongs in `roles//templates/..j2` and is consumed via `lookup('template', '..j2')` (for inline values) or `ansible.builtin.template` (for file deployment). +- The reason is reviewability and tooling. Extracted assets get proper syntax highlighting, linting, and diffing. Inline heredocs and multi-line `query:` or `command:` blocks hide all three and make the task file harder to read. +- One-liners such as a single `command:` or a short `shell:` pipeline MAY stay inline. The rule targets bodies that are themselves a program, not every shell call. + For documentation, comments, semantics, and writing guidance, see [documentation.md](../documentation.md). For test commands and testing standards, see [testing.md](../actions/testing.md). diff --git a/roles/web-app-peertube/files/dump_peertube_diagnostics.sh b/roles/web-app-peertube/files/diagnostics.sh similarity index 95% rename from roles/web-app-peertube/files/dump_peertube_diagnostics.sh rename to roles/web-app-peertube/files/diagnostics.sh index b038bdccf0..04a4701c77 100755 --- a/roles/web-app-peertube/files/dump_peertube_diagnostics.sh +++ b/roles/web-app-peertube/files/diagnostics.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -# roles/web-app-peertube/files/dump_peertube_diagnostics.sh +# roles/web-app-peertube/files/diagnostics.sh # # Best-effort diagnostics for the PeerTube container after a plugin-install # failure. Prints container state, health, resource limits, logs, stats, # host memory, and OOM-killer evidence. Always exits 0 โ€” the ansible rescue # step is responsible for re-raising the failure after the dump. # -# Usage: dump_peertube_diagnostics.sh +# Usage: diagnostics.sh # Env: PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB, PEERTUBE_MAX_OLD_SPACE_SIZE # (optional โ€” printed into the report for correlation). diff --git a/roles/web-app-peertube/filter_plugins/install_mem_limit.py b/roles/web-app-peertube/filter_plugins/install_mem_limit.py new file mode 100644 index 0000000000..c5ce522cc1 --- /dev/null +++ b/roles/web-app-peertube/filter_plugins/install_mem_limit.py @@ -0,0 +1,77 @@ +# roles/web-app-peertube/filter_plugins/install_mem_limit.py +# Derive a temporary cgroup memory ceiling for the one-off +# `npm run plugin:install` exec. +# +# Usage (Jinja): +# {{ PEERTUBE_MEM_LIMIT +# | install_mem_limit(PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB) }} +# +# Formula: +# bytes(mem_limit) + install_heap_mb * overhead * 10^6 +# +# Why `overhead` (default 4): +# `--max-old-space-size=` caps only V8 old-generation heap. Actual RSS +# during `npm install` is higher: V8 young-gen + code space, native malloc +# (node-gyp C++ builds, native bindings), and spawned postinstall workers +# with their own RSS. Empirically ~4ร— the heap cap for peertube plugin +# installs โ€” the overhead is the headroom above the main peertube process +# (which keeps running in the same cgroup via `container exec`). +# +# Output: integer bytes. `container update --memory` / `docker update --memory` +# accepts a raw byte count. + +from ansible.errors import AnsibleFilterError + +try: + from plugins.filter.node_autosize import _to_bytes +except Exception as e: + raise AnsibleFilterError( + f"Failed to import _to_bytes from plugins.filter.node_autosize: {e}" + ) + + +def install_mem_limit(mem_limit, install_heap_mb, overhead: int = 4) -> int: + """ + Return install-window cgroup memory ceiling in bytes. + + Args: + mem_limit: compose mem_limit of the long-running service + (e.g. "8g", "512m", int bytes). + install_heap_mb: V8 old-space cap used for the install exec, in MB + (decimal, matches Node's --max-old-space-size unit). + overhead: multiplier applied to install_heap_mb to cover + native RSS + spawned workers (default 4). + """ + base_bytes = _to_bytes(mem_limit) + if base_bytes is None: + raise AnsibleFilterError( + f"install_mem_limit: mem_limit is empty/None: {mem_limit!r}" + ) + + try: + heap_mb = int(install_heap_mb) + except Exception: + raise AnsibleFilterError( + f"install_mem_limit: install_heap_mb must be int-like, got: {install_heap_mb!r}" + ) + + try: + ov = int(overhead) + except Exception: + raise AnsibleFilterError( + f"install_mem_limit: overhead must be int-like, got: {overhead!r}" + ) + + if heap_mb <= 0: + raise AnsibleFilterError( + f"install_mem_limit: install_heap_mb must be > 0, got: {heap_mb}" + ) + if ov <= 0: + raise AnsibleFilterError(f"install_mem_limit: overhead must be > 0, got: {ov}") + + return base_bytes + heap_mb * ov * 10**6 + + +class FilterModule(object): + def filters(self): + return {"install_mem_limit": install_mem_limit} diff --git a/roles/web-app-peertube/tasks/main.yml b/roles/web-app-peertube/tasks/main.yml index eb5e211102..bb871230a3 100644 --- a/roles/web-app-peertube/tasks/main.yml +++ b/roles/web-app-peertube/tasks/main.yml @@ -16,9 +16,9 @@ docker_compose_flush_handlers: true - name: "Install and activate auth-openid-connect plugin if OIDC is enabled" - include_tasks: 01_enable-oidc.yml + include_tasks: oidc/enable.yml when: PEERTUBE_OIDC_ENABLED | bool - name: "Deinstall and disable auth-openid-connect plugin if OIDC is enabled" - include_tasks: 02_disable-oidc.yml + include_tasks: oidc/disable.yml when: not PEERTUBE_OIDC_ENABLED | bool diff --git a/roles/web-app-peertube/tasks/02_disable-oidc.yml b/roles/web-app-peertube/tasks/oidc/disable.yml similarity index 100% rename from roles/web-app-peertube/tasks/02_disable-oidc.yml rename to roles/web-app-peertube/tasks/oidc/disable.yml diff --git a/roles/web-app-peertube/tasks/01_enable-oidc.yml b/roles/web-app-peertube/tasks/oidc/enable.yml similarity index 54% rename from roles/web-app-peertube/tasks/01_enable-oidc.yml rename to roles/web-app-peertube/tasks/oidc/enable.yml index 5d18eea01c..622df59b3f 100644 --- a/roles/web-app-peertube/tasks/01_enable-oidc.yml +++ b/roles/web-app-peertube/tasks/oidc/enable.yml @@ -10,16 +10,30 @@ failed_when: false changed_when: false -- name: "Install auth-openid-connect plugin for Peertube (with diagnostics on failure)" +- name: "Install auth-openid-connect plugin for Peertube (with temp mem_limit bump + diagnostics on failure)" when: peertube_oidc_plugin_check.rc != 0 block: + # Raise the container cgroup ceiling for the duration of the install. + # The install node process and the long-running main peertube node + # process share this cgroup (install runs via `container exec`), and + # their combined RSS plus native allocations (node-gyp, npm postinstall) + # was tripping the runtime mem_limit and SIGKILLing peertube, surfacing + # as "Container is restarting" on subsequent exec attempts. + # --memory-swap -1 allows unlimited swap for the install window (host + # has a 16G swapfile from issue #162). + - name: "PEERTUBE | Raise mem_limit for plugin install" + ansible.builtin.command: >- + container update + --memory {{ PEERTUBE_OIDC_PLUGIN_INSTALL_MEM_LIMIT }} + --memory-swap -1 + {{ PEERTUBE_CONTAINER }} + changed_when: true + - name: "Install auth-openid-connect plugin for Peertube" - # The plugin install spawns a fresh node process under the running PeerTube - # container. Without an explicit heap cap, V8 inherits the container-wide - # NODE_OPTIONS (~35% of mem_limit, multi-GB) and can push a constrained CI - # runner over its host RAM, getting SIGKILLed (rc=137) before the install - # completes. Force a tight heap and retry to absorb transient host pressure. + # V8 heap is still capped tight โ€” the cgroup bump above covers native + # memory and spawned workers, while --max-old-space-size keeps the + # install's V8 old-generation predictable. ansible.builtin.command: > container exec -e NODE_OPTIONS=--max-old-space-size={{ PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB }} @@ -31,10 +45,25 @@ until: peertube_oidc_install.rc == 0 notify: compose up + # Restore the compose-defined mem_limit on the happy path only. + # --memory-swap is pinned to the same value (no extra swap for the + # container) โ€” matches the bump contract (extra swap granted solely + # during the install window). On failure the rescue block leaves the + # bumped limits in place so diagnostics reflect real install-time + # state; the next `compose up` recreates the container and reinstates + # docker's default memory-swap behavior from compose.yml. + - name: "PEERTUBE | Restore compose-defined mem_limit" + ansible.builtin.command: >- + container update + --memory {{ PEERTUBE_MEM_LIMIT }} + --memory-swap {{ PEERTUBE_MEM_LIMIT }} + {{ PEERTUBE_CONTAINER }} + changed_when: true + rescue: - name: "PEERTUBE | Dump diagnostics after install failure" - ansible.builtin.script: "{{ role_path }}/files/dump_peertube_diagnostics.sh {{ PEERTUBE_CONTAINER }}" + ansible.builtin.script: "{{ role_path }}/files/diagnostics.sh {{ PEERTUBE_CONTAINER }}" environment: PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB: "{{ PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB }}" PEERTUBE_MAX_OLD_SPACE_SIZE: "{{ PEERTUBE_MAX_OLD_SPACE_SIZE }}" @@ -55,17 +84,7 @@ login_password: "{{ lookup('database', application_id, 'password') }}" login_host: "{{ lookup('database', application_id, 'reach_host') }}" login_port: "{{ lookup('database', application_id, 'port') }}" - query: | - UPDATE public.plugin - SET settings = '{{ oidc_settings | to_json }}'::jsonb, - enabled = TRUE, - uninstalled = FALSE - WHERE name = 'auth-openid-connect' - AND ( - settings IS DISTINCT FROM '{{ oidc_settings | to_json }}'::jsonb - OR enabled IS DISTINCT FROM TRUE - OR uninstalled IS DISTINCT FROM FALSE - ); + query: "{{ lookup('template', 'upsert-oidc-plugin.sql.j2') }}" register: _peertube_oidc_update retries: 5 delay: 3 diff --git a/roles/web-app-peertube/templates/upsert-oidc-plugin.sql.j2 b/roles/web-app-peertube/templates/upsert-oidc-plugin.sql.j2 new file mode 100644 index 0000000000..baf63aa92e --- /dev/null +++ b/roles/web-app-peertube/templates/upsert-oidc-plugin.sql.j2 @@ -0,0 +1,10 @@ +UPDATE public.plugin +SET settings = '{{ oidc_settings | to_json }}'::jsonb, + enabled = TRUE, + uninstalled = FALSE +WHERE name = 'auth-openid-connect' + AND ( + settings IS DISTINCT FROM '{{ oidc_settings | to_json }}'::jsonb + OR enabled IS DISTINCT FROM TRUE + OR uninstalled IS DISTINCT FROM FALSE + ); diff --git a/roles/web-app-peertube/vars/main.yml b/roles/web-app-peertube/vars/main.yml index 8f18c782e0..847546dba8 100644 --- a/roles/web-app-peertube/vars/main.yml +++ b/roles/web-app-peertube/vars/main.yml @@ -15,6 +15,7 @@ PEERTUBE_OIDC_ENABLED: "{{ lookup('config', application_id, 'compose. # Performance PEERTUBE_CPUS: "{{ lookup('config', application_id, 'compose.services.peertube.cpus') | float }}" +PEERTUBE_MEM_LIMIT: "{{ lookup('config', application_id, 'compose.services.peertube.mem_limit') }}" PEERTUBE_MAX_OLD_SPACE_SIZE: "{{ applications | node_max_old_space_size(application_id, entity_name) }}" _peertube_concurrency_candidate: "{{ ((PEERTUBE_CPUS | float) * 0.5) | round(0, 'floor') | int }}" PEERTUBE_TRANSCODING_CONCURRENCY: "{{ [ ( [ (_peertube_concurrency_candidate | int), 1 ] | max ), 8 ] | min }}" @@ -28,3 +29,16 @@ PEERTUBE_TRANSCODING_CONCURRENCY: "{{ [ ( [ (_peertube_concurrency_candidate | i # capped at 1024 MB to avoid tripping the host OOM-killer on constrained CI # runners (see issue #162). PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB: "{{ applications | node_max_old_space_size(application_id, entity_name, pct=0.125, min_mb=256, hardcap_mb=1024) }}" + +# Temporary cgroup memory ceiling (bytes) during the one-off `npm run +# plugin:install`. Raised above compose.services.peertube.mem_limit so the +# install process and the long-running main peertube node process (both +# sharing the same cgroup via `container exec`) do not collectively trip the +# runtime mem_limit and get SIGKILLed โ€” which was manifesting as "Container +# is restarting" on the plugin install exec. Restored to the compose-defined +# value by the happy-path tail of the block in tasks/oidc/enable.yml. +# +# Formula: mem_limit_bytes + install_heap_mb * overhead * 10^6 +# overhead=4 matches the empirical heap-to-RSS ratio for `npm install` with +# native deps (node-gyp, postinstall workers each carry their own RSS). +PEERTUBE_OIDC_PLUGIN_INSTALL_MEM_LIMIT: "{{ PEERTUBE_MEM_LIMIT | install_mem_limit(PEERTUBE_OIDC_PLUGIN_INSTALL_MAX_OLD_SPACE_MB) }}" diff --git a/tests/unit/roles/web-app-peertube/__init__.py b/tests/unit/roles/web-app-peertube/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/roles/web-app-peertube/filter_plugins/__init__.py b/tests/unit/roles/web-app-peertube/filter_plugins/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/roles/web-app-peertube/filter_plugins/test_install_mem_limit.py b/tests/unit/roles/web-app-peertube/filter_plugins/test_install_mem_limit.py new file mode 100644 index 0000000000..fd8d8f921c --- /dev/null +++ b/tests/unit/roles/web-app-peertube/filter_plugins/test_install_mem_limit.py @@ -0,0 +1,112 @@ +import unittest +import pathlib +import importlib.util + +from ansible.errors import AnsibleFilterError + + +def _load_install_mem_limit_module(): + """ + Load the install_mem_limit.py filter plugin module from the + roles/web-app-peertube path, without requiring the roles directory to + be a Python package. + """ + # tests/unit/roles/web-app-peertube/filter_plugins/test_install_mem_limit.py + test_file = pathlib.Path(__file__).resolve() + repo_root = test_file.parents[5] + + plugin_path = ( + repo_root + / "roles" + / "web-app-peertube" + / "filter_plugins" + / "install_mem_limit.py" + ) + + if not plugin_path.is_file(): + raise RuntimeError(f"Could not find install_mem_limit.py at {plugin_path}") + + spec = importlib.util.spec_from_file_location("install_mem_limit", plugin_path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +_mod = _load_install_mem_limit_module() +install_mem_limit = _mod.install_mem_limit +FilterModule = _mod.FilterModule + + +class TestInstallMemLimitFilter(unittest.TestCase): + def test_registry_contains_filter(self): + self.assertIn("install_mem_limit", FilterModule().filters()) + + def test_default_overhead_with_string_mem_limit(self): + # "8g" = 8_000_000_000 bytes (decimal, matches _to_bytes semantics) + # install_heap_mb=1024, overhead=4 -> 1024 * 4 * 10^6 = 4_096_000_000 + # total = 12_096_000_000 + self.assertEqual(install_mem_limit("8g", 1024), 12_096_000_000) + + def test_int_mem_limit_is_treated_as_bytes(self): + self.assertEqual(install_mem_limit(8_000_000_000, 1024), 12_096_000_000) + + def test_custom_overhead(self): + # "8g" + 1024 * 2 * 10^6 = 10_048_000_000 + self.assertEqual(install_mem_limit("8g", 1024, overhead=2), 10_048_000_000) + + def test_small_mem_limit(self): + # "512m" = 512_000_000; 256 MB heap * 4 = 1_024_000_000 + # total = 1_536_000_000 + self.assertEqual(install_mem_limit("512m", 256), 1_536_000_000) + + def test_binary_suffix_gib(self): + # "1Gib" = 1 GiB = 1_073_741_824; 128 * 4 * 10^6 = 512_000_000 + self.assertEqual(install_mem_limit("1Gib", 128), 1_073_741_824 + 512_000_000) + + def test_string_int_install_heap_mb_accepted(self): + # Ansible Jinja often passes stringly-typed numbers + self.assertEqual(install_mem_limit("8g", "1024"), 12_096_000_000) + + def test_none_mem_limit_raises(self): + with self.assertRaises(AnsibleFilterError): + install_mem_limit(None, 1024) + + def test_empty_mem_limit_raises(self): + with self.assertRaises(AnsibleFilterError): + install_mem_limit("", 1024) + + def test_unparseable_mem_limit_raises(self): + with self.assertRaises(AnsibleFilterError): + install_mem_limit("not-a-size", 1024) + + def test_non_int_install_heap_mb_raises(self): + with self.assertRaises(AnsibleFilterError): + install_mem_limit("8g", "not-a-number") + + def test_non_int_overhead_raises(self): + with self.assertRaises(AnsibleFilterError): + install_mem_limit("8g", 1024, overhead="lots") + + def test_zero_install_heap_mb_raises(self): + with self.assertRaises(AnsibleFilterError): + install_mem_limit("8g", 0) + + def test_negative_install_heap_mb_raises(self): + with self.assertRaises(AnsibleFilterError): + install_mem_limit("8g", -1) + + def test_zero_overhead_raises(self): + with self.assertRaises(AnsibleFilterError): + install_mem_limit("8g", 1024, overhead=0) + + def test_negative_overhead_raises(self): + with self.assertRaises(AnsibleFilterError): + install_mem_limit("8g", 1024, overhead=-1) + + def test_result_is_int(self): + self.assertIsInstance(install_mem_limit("8g", 1024), int) + + +if __name__ == "__main__": + unittest.main()