diff --git a/Makefile b/Makefile index 8c8980f023..8e4207fe9b 100644 --- a/Makefile +++ b/Makefile @@ -889,6 +889,7 @@ SWARM_ARCH = $(or $(arch),$(shell bash scripts/meta/resolve/architecture.sh)) # Param arch: architecture the cluster deploys on, amd64 | arm64 (default: the host's own). # Param network: optional node network mode pinned on the row (clearnet | tor | multi); empty keeps the run's rotation. # Param step_timeout: optional minute budget for the matrix-deploy step (default 690). +# Param INFINITO_INVENTORY_EXTRA_VARS: optional environment JSON object laid over every round's inventory vars, `applications` merged per app; must not contain ';'. # Note: With keep=true use `make swarm-exec` / `make swarm-shell` to inspect, `make swarm-down` to release. swarm-zombie: install-act @test -n '$(app)' || { echo 'usage: make swarm-zombie app= [distros=] [variant=] [name=] [disable=] [network=] [keep=true]'; exit 2; } @@ -905,6 +906,7 @@ swarm-zombie: install-act INFINITO_CACHE_HOST_PACMAN_PATH=$${INFINITO_CACHE_HOST_PACMAN_PATH:-}; \ INFINITO_APP_DISCOVERY_RUNNER=host; \ INFINITO_DEPLOY_MODE=swarm; \ + INFINITO_INVENTORY_EXTRA_VARS=$${INFINITO_INVENTORY_EXTRA_VARS:-}; \ disable=$(disable); \ SWARM_NAME=$(or $(name),$(app)); \ INFINITO_SWARM_STEP_TIMEOUT_MINUTES=$(or $(step_timeout),690); \ diff --git a/tests/unit/python/scripts/tests/deploy/act/test_workflow.py b/tests/unit/python/scripts/tests/deploy/act/test_workflow.py new file mode 100644 index 0000000000..c79bddf97c --- /dev/null +++ b/tests/unit/python/scripts/tests/deploy/act/test_workflow.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "scripts" / "tests" / "deploy" / "act" / "workflow.sh" +ACT_STUB = '#!/usr/bin/env bash\nprintf \'%s\\0\' "$@" >"${STUB_LOG}"\n' +EXTRA_VARS = json.dumps( + { + "applications": { + "web-app-erpnext": {"services": {"erpnext": {"server_scripts": True}}}, + "web-app-cst-demo": {"billing": {"bank": "Verify Bank, IBAN DE00 0000"}}, + } + } +) + + +def act_argv(act_env: str) -> list[str]: + """Run the act wrapper against an ``act`` stub and return the argv it got. + + Args: + act_env: value of ``ACT_ENV``. + + Returns: + The arguments the stub received, one list item per argument. + """ + with tempfile.TemporaryDirectory() as tmp: + stub = Path(tmp) / "act" + stub.write_text(ACT_STUB) + stub.chmod(0o755) + workflow = Path(tmp) / "call-test-deploy.yml" + workflow.write_text("on: workflow_dispatch\n") + log = Path(tmp) / "act.argv" + env = {k: v for k, v in os.environ.items() if k != "BASH_ENV"} + env.update( + PATH=f"{tmp}{os.pathsep}{os.environ['PATH']}", + STUB_LOG=str(log), + ACT_WORKFLOW=str(workflow), + ACT_ENV=act_env, + ACT_ACTION_CACHE_PATH=str(Path(tmp) / "actcache"), + ACT_PLATFORM_IMAGE="catthehacker/ubuntu:act-latest", + INFINITO_CONTAINER="infinito-test", + ) + subprocess.run( + ["bash", str(SCRIPT)], + env=env, + capture_output=True, + text=True, + check=True, + ) + return log.read_text().split("\0")[:-1] # nocheck: cache-read stub argv log + + +def env_items(argv: list[str]) -> list[str]: + """Collect the values passed behind each ``--env`` flag. + + Args: + argv: arguments the act stub received. + + Returns: + Every ``--env`` value in order. + """ + return [argv[i + 1] for i, arg in enumerate(argv) if arg == "--env"] + + +class TestActWorkflowEnv(unittest.TestCase): + def test_json_with_spaces_reaches_act_as_one_env_item(self) -> None: + items = env_items( + act_argv( + "INFINITO_DEPLOY_MODE=swarm; " + f"INFINITO_INVENTORY_EXTRA_VARS={EXTRA_VARS}; " + "disable=matomo" + ) + ) + self.assertEqual( + items, + [ + "INFINITO_DEPLOY_MODE=swarm", + f"INFINITO_INVENTORY_EXTRA_VARS={EXTRA_VARS}", + "disable=matomo", + ], + ) + + def test_an_empty_value_still_passes_as_its_own_item(self) -> None: + items = env_items(act_argv("INFINITO_INVENTORY_EXTRA_VARS=; disable=css")) + self.assertEqual(items, ["INFINITO_INVENTORY_EXTRA_VARS=", "disable=css"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/tests/swarm/test_extra_vars.py b/tests/unit/python/utils/tests/swarm/test_extra_vars.py new file mode 100644 index 0000000000..aac0159a2b --- /dev/null +++ b/tests/unit/python/utils/tests/swarm/test_extra_vars.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import json +import unittest +from unittest import mock + +from utils.tests.swarm import extra_vars + +ROUND_OVERRIDES = { + "applications": { + "web-app-erpnext": {"services": {"backup": {"remote": ["10.0.0.2"]}}} + }, + "users": {"administrator": {"authorized_keys": ["ssh-ed25519 AAAA"]}}, + "STORAGE_CONSTRAINED": False, +} + + +class TestInventoryExtraVars(unittest.TestCase): + def test_unset_or_empty_variable_keeps_the_round_overrides(self): + for env in ({}, {"INFINITO_INVENTORY_EXTRA_VARS": ""}): + with mock.patch.dict(extra_vars.os.environ, env, clear=True): + self.assertEqual( + extra_vars.with_extra_vars(ROUND_OVERRIDES), ROUND_OVERRIDES + ) + + def test_nested_app_switch_merges_beside_the_round_overrides(self): + extra = { + "applications": { + "web-app-erpnext": {"services": {"erpnext": {"server_scripts": True}}} + }, + "STORAGE_CONSTRAINED": True, + } + with mock.patch.dict( + extra_vars.os.environ, + {"INFINITO_INVENTORY_EXTRA_VARS": json.dumps(extra)}, + clear=True, + ): + merged = extra_vars.with_extra_vars(ROUND_OVERRIDES) + self.assertEqual( + merged["applications"]["web-app-erpnext"]["services"], + {"backup": {"remote": ["10.0.0.2"]}, "erpnext": {"server_scripts": True}}, + ) + self.assertEqual(merged["users"], ROUND_OVERRIDES["users"]) + self.assertTrue(merged["STORAGE_CONSTRAINED"]) + + def test_a_non_object_variable_fails_loud(self): + with ( + mock.patch.dict( + extra_vars.os.environ, + {"INFINITO_INVENTORY_EXTRA_VARS": "[]"}, + clear=True, + ), + self.assertRaises(SystemExit), + ): + extra_vars.with_extra_vars(ROUND_OVERRIDES) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/tests/swarm/extra_vars.py b/utils/tests/swarm/extra_vars.py new file mode 100644 index 0000000000..0e2e429ef3 --- /dev/null +++ b/utils/tests/swarm/extra_vars.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import json +import os + +from utils.cache.base import _deep_merge + + +def with_extra_vars(overrides: dict) -> dict: + """Lay INFINITO_INVENTORY_EXTRA_VARS over a round's inventory overrides. + + Args: + overrides: the round's overrides (`applications`, `users`, ...). + + Returns: + The overrides unchanged when the variable is unset or empty, otherwise + the JSON object merged in recursively, the variable winning per leaf. + """ + raw = os.environ.get("INFINITO_INVENTORY_EXTRA_VARS") + if not raw: + return overrides + extra = json.loads(raw) + if not isinstance(extra, dict): + raise SystemExit( + "swarm-matrix: INFINITO_INVENTORY_EXTRA_VARS must be a JSON object" + ) + return _deep_merge(overrides, extra) diff --git a/utils/tests/swarm/matrix.py b/utils/tests/swarm/matrix.py index 73d9a25fd8..d742c626d4 100644 --- a/utils/tests/swarm/matrix.py +++ b/utils/tests/swarm/matrix.py @@ -14,6 +14,7 @@ from utils import PROJECT_ROOT from utils.storage.constrained import host_storage_constrained from utils.tests.swarm.derive_includes import derive_includes, variant_scope from utils.tests.swarm.extend_inventory import drop_backup_network_mode, mesh_enabled +from utils.tests.swarm.extra_vars import with_extra_vars from utils.tests.swarm.mesh import ( bootstrap_mesh, converge_mesh, @@ -414,16 +415,17 @@ def main(argv: list[str] | None = None) -> int: applications = backup_applications_overrides(providers) if meshed: applications = merge_with_defaults(applications, layout_applications()) - vars_payload = _bake_overrides( - base_overrides={ - "applications": applications, - "users": { - name: {"authorized_keys": [key]} for name, key in pubkeys.items() - }, - "STORAGE_CONSTRAINED": host_storage_constrained( - [app_id], round_variants, local_vantage="/" - ), + base_overrides = { + "applications": applications, + "users": { + name: {"authorized_keys": [key]} for name, key in pubkeys.items() }, + "STORAGE_CONSTRAINED": host_storage_constrained( + [app_id], round_variants, local_vantage="/" + ), + } + vars_payload = _bake_overrides( + base_overrides=with_extra_vars(base_overrides), variant_payloads=variant_payloads, ) extras_path = f"{inv_root}/swarm-nfs-extras.yml"