From 0d18d86243a100d990c18fa613b7a8254cbdf30d Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 4 Dec 2025 23:48:43 +0100 Subject: [PATCH] Add --vars support to inventory creation, implement deep JSON overrides for host_vars, and update CI workflow to pass MASK_CREDENTIALS_IN_LOGS=false. Includes: - New apply_vars_overrides() with deep merge logic - New --vars CLI argument in cli/create/inventory.py - Added unit tests for vars handling in test_inventory.py - Updated test-deploy workflow to pass --vars in all deploy phases Ref: ChatGPT conversation https://chatgpt.com/share/69320f49-6c00-800f-8875-49d36935ae3a --- .github/workflows/test-deploy.yml | 3 + cli/create/inventory.py | 83 +++++++++++++++++++++++++ tests/unit/cli/create/test_inventory.py | 80 ++++++++++++++++++++++++ 3 files changed, 166 insertions(+) diff --git a/.github/workflows/test-deploy.yml b/.github/workflows/test-deploy.yml index 85122a8c..fa2ba3e5 100644 --- a/.github/workflows/test-deploy.yml +++ b/.github/workflows/test-deploy.yml @@ -48,6 +48,7 @@ jobs: run: | python -m cli.deploy.container run --image "$INFINITO_IMAGE" --build -- \ --exclude "$EXCLUDED_ROLES" \ + --vars '{"MASK_CREDENTIALS_IN_LOGS": false}' \ --authorized-keys "ssh-ed25519 AAAA_TEST_DUMMY_KEY github-ci-dummy@infinito" \ -- \ -T server \ @@ -60,6 +61,7 @@ jobs: run: | python -m cli.deploy.container run --image "$INFINITO_IMAGE" -- \ --exclude "$EXCLUDED_ROLES" \ + --vars '{"MASK_CREDENTIALS_IN_LOGS": false}' \ --authorized-keys "ssh-ed25519 AAAA_TEST_DUMMY_KEY github-ci-dummy@infinito" \ -- \ -T server \ @@ -73,6 +75,7 @@ jobs: run: | python -m cli.deploy.container run --image "$INFINITO_IMAGE" -- \ --exclude "$EXCLUDED_ROLES" \ + --vars '{"MASK_CREDENTIALS_IN_LOGS": false}' \ --authorized-keys "ssh-ed25519 AAAA_TEST_DUMMY_KEY github-ci-dummy@infinito" \ -- \ -T server \ diff --git a/cli/create/inventory.py b/cli/create/inventory.py index ab5315a1..7b7d2c5a 100644 --- a/cli/create/inventory.py +++ b/cli/create/inventory.py @@ -38,6 +38,7 @@ import concurrent.futures import os import secrets import string +import json try: import yaml @@ -76,6 +77,69 @@ def run_subprocess( raise SystemExit(msg) return result +def deep_update_commented_map(target: CommentedMap, updates: Dict[str, Any]) -> None: + """ + Recursively merge updates into a ruamel CommentedMap. + + - If a value in updates is a mapping, it is merged into the existing mapping. + - Non-mapping values overwrite existing values. + """ + for key, value in updates.items(): + if isinstance(value, dict): + existing = target.get(key) + if not isinstance(existing, CommentedMap): + existing = CommentedMap() + target[key] = existing + deep_update_commented_map(existing, value) + else: + target[key] = value + + +def apply_vars_overrides(host_vars_file: Path, json_str: str) -> None: + """ + Apply JSON overrides to host_vars/.yml. + + Behavior: + - json_str must contain a JSON object at the top level. + - All keys in that object (possibly nested) are merged into the + existing document. + - Existing values are overwritten by values from the JSON. + - Non-existing keys are created. + + Example: + --vars '{"SSL_ENABLED": false, "networks": {"internet": {"ip4": "10.0.0.10"}}}' + """ + try: + overrides = json.loads(json_str) + except json.JSONDecodeError as exc: + raise SystemExit(f"Invalid JSON passed to --vars: {exc}") from exc + + if not isinstance(overrides, dict): + raise SystemExit("JSON for --vars must be an object at the top level.") + + yaml_rt = YAML(typ="rt") + yaml_rt.preserve_quotes = True + + if host_vars_file.exists(): + with host_vars_file.open("r", encoding="utf-8") as f: + doc = yaml_rt.load(f) + if doc is None: + doc = CommentedMap() + else: + doc = CommentedMap() + + if not isinstance(doc, CommentedMap): + tmp = CommentedMap() + for k, v in dict(doc).items(): + tmp[k] = v + doc = tmp + + deep_update_commented_map(doc, overrides) + + host_vars_file.parent.mkdir(parents=True, exist_ok=True) + with host_vars_file.open("w", encoding="utf-8") as f: + yaml_rt.dump(doc, f) + def build_env_with_project_root(project_root: Path) -> Dict[str, str]: """ @@ -892,6 +956,16 @@ def main(argv: Optional[List[str]] = None) -> None: "under the inventory directory; missing keys are appended." ), ) + parser.add_argument( + "--vars", + required=False, + help=( + "Optional JSON string with additional values for host_vars/.yml. " + "The JSON must have an object at the top level. All keys from this " + "object (including nested ones) are merged into host_vars and " + "overwrite existing values." + ), + ) parser.add_argument( "--ip4", default="127.0.0.1", @@ -1082,6 +1156,15 @@ def main(argv: Optional[List[str]] = None) -> None: project_root=project_root, workers=args.workers, ) + if args.vars: + print( + f"[INFO] Applying JSON overrides to host_vars for host '{args.host}' " + f"via --vars" + ) + apply_vars_overrides( + host_vars_file=host_vars_file, + json_str=args.vars, + ) print("[INFO] Done. Inventory and host_vars updated without deleting existing values.") diff --git a/tests/unit/cli/create/test_inventory.py b/tests/unit/cli/create/test_inventory.py index da455d08..8dea093d 100644 --- a/tests/unit/cli/create/test_inventory.py +++ b/tests/unit/cli/create/test_inventory.py @@ -3,6 +3,7 @@ import sys import tempfile import unittest from pathlib import Path +import yaml # Make cli module importable (same pattern as test_credentials.py) dir_path = os.path.abspath( @@ -19,6 +20,7 @@ from cli.create.inventory import ( # type: ignore filter_inventory_by_ignore, get_path_administrator_home_from_group_vars, ensure_administrator_authorized_keys, + apply_vars_overrides, ) from ruamel.yaml import YAML @@ -505,6 +507,84 @@ existing_key: foo self.assertIn(key1, lines) self.assertIn(key2, lines) + def test_apply_vars_overrides_sets_top_level_flag(self): + """ + apply_vars_overrides() should create the host_vars file (if missing) + and set a simple top-level flag like MASK_CREDENTIALS_IN_LOGS: false. + """ + with tempfile.TemporaryDirectory() as tmpdir: + host_vars_file = Path(tmpdir) / "host_vars.yml" + + # File should not exist initially + self.assertFalse(host_vars_file.exists()) + + json_payload = '{"MASK_CREDENTIALS_IN_LOGS": false}' + apply_vars_overrides(host_vars_file, json_payload) + + # File must now exist and contain the flag as a boolean + self.assertTrue(host_vars_file.exists()) + with host_vars_file.open("r", encoding="utf-8") as f: + data = yaml.safe_load(f) or {} + + self.assertIn("MASK_CREDENTIALS_IN_LOGS", data) + self.assertIs(data["MASK_CREDENTIALS_IN_LOGS"], False) + + def test_apply_vars_overrides_nested_merge_and_overwrite(self): + """ + apply_vars_overrides() must overwrite nested values but preserve + unrelated keys, effectively doing a deep merge. + """ + with tempfile.TemporaryDirectory() as tmpdir: + host_vars_file = Path(tmpdir) / "host_vars_nested.yml" + + original = { + "networks": { + "internet": { + "ip4": "1.2.3.4", + "ip6": "::1", + } + }, + "SSL_ENABLED": True, + } + host_vars_file.write_text( + yaml.safe_dump(original), + encoding="utf-8", + ) + + json_payload = """ + { + "networks": { + "internet": { + "ip4": "10.0.0.10" + } + }, + "SSL_ENABLED": false + } + """ + apply_vars_overrides(host_vars_file, json_payload) + + with host_vars_file.open("r", encoding="utf-8") as f: + data = yaml.safe_load(f) or {} + + # Nested merge: ip4 overwritten, ip6 preserved + self.assertEqual(data["networks"]["internet"]["ip4"], "10.0.0.10") + self.assertEqual(data["networks"]["internet"]["ip6"], "::1") + + # Top-level boolean flag overwritten + self.assertIs(data["SSL_ENABLED"], False) + + def test_apply_vars_overrides_requires_object(self): + """ + apply_vars_overrides() must reject JSON that does not contain an + object at the top level (e.g. an array) and exit with SystemExit. + """ + with tempfile.TemporaryDirectory() as tmpdir: + host_vars_file = Path(tmpdir) / "host_vars_invalid.yml" + + invalid_json = '["not-an-object"]' + with self.assertRaises(SystemExit): + apply_vars_overrides(host_vars_file, invalid_json) + if __name__ == "__main__": unittest.main()