Refactor setup workflow and make install robust via virtualenv

- Introduce a dedicated Python virtualenv (deps target) and run all setup scripts through it
- Fix missing PyYAML errors in clean, CI, and Nix environments
- Refactor build defaults into cli/setup for clearer semantics
- Make setup deterministic and independent from system Python
- Replace early Makefile shell expansion with runtime evaluation
- Rename messy-test to test-messy and update deploy logic and tests accordingly
- Keep setup and test targets consistent across Makefile, CLI, and unit tests

https://chatgpt.com/share/693de226-00ac-800f-8cbd-06552b2f283c

Makefile

@@ -1,12 +1,13 @@
+SHELL 				:= /usr/bin/env bash
+VENV        		?= .venv
+PYTHON      		:= $(VENV)/bin/python
+PIP         		:= $(PYTHON) -m pip
 ROLES_DIR           := ./roles
 APPLICATIONS_OUT    := ./group_vars/all/04_applications.yml
-APPLICATIONS_SCRIPT := ./cli/build/defaults/applications.py
+APPLICATIONS_SCRIPT := ./cli/setup/applications.py
+USERS_SCRIPT        := ./cli/setup/users.py
 USERS_OUT           := ./group_vars/all/03_users.yml
-USERS_SCRIPT        := ./cli/build/defaults/users.py
 INCLUDES_SCRIPT     := ./cli/build/role_include.py
-PYTHON  			?= python3
-
-INCLUDE_GROUPS = $(shell $(PYTHON) main.py meta categories invokable -s "-" --no-signal | tr '\n' ' ')
 
 # Directory where these include-files will be written
 INCLUDES_OUT_DIR    := ./tasks/groups
@@ -20,7 +21,7 @@ RESERVED_USERNAMES := $(shell \
     | paste -sd, - \
 )
 
-.PHONY: build install test
+.PHONY: deps setup setup-clean test-messy test install
 
 clean-keep-logs:
 	@echo "🧹 Cleaning ignored files but keeping logs/…"
@@ -46,7 +47,7 @@ dockerignore:
 	cat .gitignore > .dockerignore
 	echo ".git" >> .dockerignore 
 
-messy-build: dockerignore
+setup: deps dockerignore
 	@echo "🔧 Generating users defaults → $(USERS_OUT)…"
 	$(PYTHON) $(USERS_SCRIPT) \
 	  --roles-dir $(ROLES_DIR) \
@@ -62,25 +63,35 @@ messy-build: dockerignore
 
 	@echo "🔧 Generating role-include files for each group…"
 	@mkdir -p $(INCLUDES_OUT_DIR)
-	@$(foreach grp,$(INCLUDE_GROUPS), \
-	  out=$(INCLUDES_OUT_DIR)/$(grp)roles.yml; \
-	  echo "→ Building $$out (pattern: '$(grp)')…"; \
-	  $(PYTHON) $(INCLUDES_SCRIPT) $(ROLES_DIR) \
-	    -p $(grp) -o $$out; \
+	@INCLUDE_GROUPS="$$( $(PYTHON) main.py meta categories invokable -s "-" --no-signal | tr '\n' ' ' )"; \
+	for grp in $$INCLUDE_GROUPS; do \
+	  out="$(INCLUDES_OUT_DIR)/$${grp}roles.yml"; \
+	  echo "→ Building $$out (pattern: '$$grp')…"; \
+	  $(PYTHON) $(INCLUDES_SCRIPT) $(ROLES_DIR) -p $$grp -o $$out; \
 	  echo "  ✅ $$out"; \
-	)
+	done
 
-messy-test: 
+setup-clean: clean setup
+	@echo "Full build with cleanup before was executed."
+
+test-messy: 
 	@echo "🧪 Running Python tests…"
 	PYTHONPATH=. $(PYTHON) -m unittest discover -s tests
 	@echo "📑 Checking Ansible syntax…"
 	ansible-playbook -i localhost, -c local $(foreach f,$(wildcard group_vars/all/*.yml),-e @$(f)) playbook.yml --syntax-check
 
-install: build
-	@echo "⚙️  Install complete."
+test: setup-clean test-messy
+	@echo "Full test with setup-clean before was executed."
 
-build: clean messy-build
-	@echo "Full build with cleanup before was executed."
+deps:
+	@if [ ! -x "$(PYTHON)" ]; then \
+		echo "🐍 Creating virtualenv $(VENV)"; \
+		python3 -m venv $(VENV); \
+	fi
+	@echo "📦 Installing Python dependencies"
+	@$(PIP) install --upgrade pip setuptools wheel
+	@$(PIP) install -e .
+
+install: deps
+	@echo "✅ Python environment installed (editable)."
 
-test: build messy-test
-	@echo "Full test with build before was executed."

cli/deploy/dedicated.py

@@ -95,8 +95,8 @@ def run_ansible_playbook(
     # 4) Test Phase
     # ---------------------------------------------------------
     if not skip_tests:
-        print("\n🧪 Running tests (make messy-test)...\n")
-        subprocess.run(["make", "messy-test"], check=True)
+        print("\n🧪 Running tests (make test-messy)...\n")
+        subprocess.run(["make", "test-messy"], check=True)
     else:
         print("\n🧪 Tests skipped (--skip-tests)\n")
 

main.py

@@ -209,7 +209,7 @@ def print_global_help(available, cli_dir):
         Fore.CYAN
     ))
     print(color_text(
-        "    corresponds to `cli/build/defaults/users.py`.",
+        "    corresponds to `cli/setup/users.py`.",
         Fore.CYAN
     ))
     print()

tests/unit/cli/deploy/test_dedicated.py

@@ -234,8 +234,8 @@ class TestRunAnsiblePlaybook(unittest.TestCase):
             "Expected 'make messy-build' when skip_build=False",
         )
         self.assertTrue(
-            any(call == ["make", "messy-test"] for call in calls),
-            "Expected 'make messy-test' when skip_tests=False",
+            any(call == ["make", "test-messy"] for call in calls),
+            "Expected 'make test-messy' when skip_tests=False",
         )
         self.assertTrue(
             any(
@@ -330,7 +330,7 @@ class TestRunAnsiblePlaybook(unittest.TestCase):
         # No cleanup, no build, no tests, no inventory validation
         self.assertFalse(any(call == ["make", "clean"] for call in calls))
         self.assertFalse(any(call == ["make", "messy-build"] for call in calls))
-        self.assertFalse(any(call == ["make", "messy-test"] for call in calls))
+        self.assertFalse(any(call == ["make", "test-messy"] for call in calls))
         self.assertFalse(
             any(
                 isinstance(call, list)

tests/unit/cli/setup/test_applications.py

@@ -10,7 +10,7 @@ import subprocess
 class TestGenerateDefaultApplications(unittest.TestCase):
     def setUp(self):
         # Path to the generator script under test
-        self.script_path = Path(__file__).resolve().parents[5] / "cli" / "build" / "defaults" / "applications.py"
+        self.script_path = Path(__file__).resolve().parents[4] / "cli" / "setup" / "applications.py"
         # Create temp role structure
         self.temp_dir = Path(tempfile.mkdtemp())
         self.roles_dir = self.temp_dir / "roles"
@@ -32,7 +32,7 @@ class TestGenerateDefaultApplications(unittest.TestCase):
         shutil.rmtree(self.temp_dir)
 
     def test_script_generates_expected_yaml(self):
-        script_path = Path(__file__).resolve().parent.parent.parent.parent.parent.parent / "cli/build/defaults/applications.py"
+        script_path = Path(__file__).resolve().parent.parent.parent.parent.parent.parent / "cli/setup/applications.py"
 
         result = subprocess.run(
             [

tests/unit/cli/setup/test_applications_and_users.py

@@ -45,7 +45,7 @@ class TestGenerateDefaultApplicationsUsers(unittest.TestCase):
         When a users.yml exists with defined users, the script should inject a 'users'
         mapping in the generated YAML, mapping each username to a Jinja2 reference.
         """
-        script_path = Path(__file__).resolve().parents[5] / "cli" / "build/defaults/applications.py"
+        script_path = Path(__file__).resolve().parents[4] / "cli" / "setup/applications.py"
         result = subprocess.run([
             "python3", str(script_path),
             "--roles-dir", str(self.roles_dir),

tests/unit/cli/setup/test_users.py

@@ -7,7 +7,7 @@ import yaml
 from collections import OrderedDict
 
 # Add cli/ to import path
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..", "cli/build/defaults/")))
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..", "cli/setup/")))
 
 import users
 
@@ -159,7 +159,7 @@ class TestGenerateUsers(unittest.TestCase):
             out_file = tmpdir / "users.yml"
 
             # Resolve script path like in other tests (relative to repo root)
-            script_path = Path(__file__).resolve().parents[5] / "cli" / "build" / "defaults" / "users.py"
+            script_path = Path(__file__).resolve().parents[4] / "cli" / "setup" / "users.py"
 
             # Run generator
             result = subprocess.run(
@@ -215,7 +215,7 @@ class TestGenerateUsers(unittest.TestCase):
                     yaml.safe_dump({"users": users_map}, f)
 
             out_file = tmpdir / "users.yml"
-            script_path = Path(__file__).resolve().parents[5] / "cli" / "build" / "defaults" / "users.py"
+            script_path = Path(__file__).resolve().parents[5] / "cli" / "setup" / "users.py"
 
             # First run
             r1 = subprocess.run(
@@ -303,7 +303,7 @@ class TestGenerateUsers(unittest.TestCase):
                 )
 
             out_file = tmpdir / "users.yml"
-            script_path = Path(__file__).resolve().parents[5] / "cli" / "build" / "defaults" / "users.py"
+            script_path = Path(__file__).resolve().parents[5] / "cli" / "setup" / "users.py"
 
             result = subprocess.run(
                 [

tests/unit/test_main.py

@@ -69,7 +69,7 @@ class TestMainHelpers(unittest.TestCase):
         """
         available = [
             (None, "deploy"),
-            ("build/defaults", "users"),
+            ("setup", "users"),
         ]
 
         main.show_full_help_for_all("/fake/cli", available)