diff --git a/.github/workflows/images-mirror-all.yml b/.github/workflows/images-mirror-all.yml index 4ee4d32377..a3a5b43e12 100644 --- a/.github/workflows/images-mirror-all.yml +++ b/.github/workflows/images-mirror-all.yml @@ -87,9 +87,7 @@ jobs: cache-dependency-path: pyproject.toml - name: 📦 Install Python deps - run: | - python -m pip install --upgrade pip - python -m pip install PyYAML + run: PYTHON=python bash scripts/install/python.sh - name: 📦 Install skopeo run: ./scripts/install/apt.sh skopeo diff --git a/.github/workflows/images-mirror-missing.yml b/.github/workflows/images-mirror-missing.yml index cf619a2416..40f77620d8 100644 --- a/.github/workflows/images-mirror-missing.yml +++ b/.github/workflows/images-mirror-missing.yml @@ -116,9 +116,7 @@ jobs: cache-dependency-path: pyproject.toml - name: 📦 Install Python deps - run: | - python -m pip install --upgrade pip - python -m pip install PyYAML + run: PYTHON=python bash scripts/install/python.sh - name: 📦 Install skopeo run: ./scripts/install/apt.sh skopeo diff --git a/.github/workflows/test-instructions.yml b/.github/workflows/test-instructions.yml index 8ab0492749..b58cf90bdf 100644 --- a/.github/workflows/test-instructions.yml +++ b/.github/workflows/test-instructions.yml @@ -68,7 +68,7 @@ jobs: if: ${{ env.ACT }} shell: bash run: | - pip install --quiet --break-system-packages pyyaml + PYTHON=python3 PIP_BREAK_SYSTEM_PACKAGES=1 bash scripts/install/python.sh apt-get update -qq && apt-get install -y -qq gettext-base jq - name: 🎲 Select the role and deploy mode diff --git a/Makefile b/Makefile index 3a03f182f2..5343ef0cdc 100644 --- a/Makefile +++ b/Makefile @@ -349,7 +349,7 @@ install-lint-force: .PHONY: install-python # Install Python tooling. install-python: install-venv - @bash scripts/install/python.sh + @bash scripts/install/python.sh deploy .PHONY: install-python-dev # Install Python tooling including lint and dev dependencies. diff --git a/pyproject.toml b/pyproject.toml index 3e7d654c81..f8581d4435 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,19 +11,21 @@ requires-python = ">=3.11" license = { file = "LICENSE.md" } dependencies = [ - "numpy", - "ansible", - "colorscheme-generator @ https://github.com/kevinveenbirkenbach/colorscheme-generator/archive/refs/tags/v0.3.0.zip", "bcrypt", - "ruamel.yaml", - "PyYAML", - "tld", - "passlib", - "requests", + "cryptography", "humanfriendly", + "PyYAML", + "ruamel.yaml", ] [project.optional-dependencies] +deploy = [ + "ansible", + "colorscheme-generator @ https://github.com/kevinveenbirkenbach/colorscheme-generator/archive/refs/tags/v0.3.0.zip", + "tld", + "passlib", + "requests", +] dev = [ "ruff==0.16.0", "import-linter", diff --git a/scripts/github/update/install_python.sh b/scripts/github/update/install_python.sh index 554956736d..199fda20c5 100755 --- a/scripts/github/update/install_python.sh +++ b/scripts/github/update/install_python.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -python3 -m pip install --upgrade pip -python3 -m pip install PyYAML +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +PYTHON=python3 bash "${SCRIPT_DIR}/../../install/python.sh" diff --git a/scripts/install/all.sh b/scripts/install/all.sh index e1dc554561..6f4a6a3542 100755 --- a/scripts/install/all.sh +++ b/scripts/install/all.sh @@ -57,7 +57,7 @@ fi bash "${INFINITO_PYTHON_INSTALL_SCRIPT}" ensure bash scripts/install/venv.sh -bash scripts/install/python.sh +bash scripts/install/python.sh deploy ANSIBLE_COLLECTIONS_DIR="${HOME}/.ansible/collections" bash scripts/install/ansible.sh stamp_dir="$(dirname "${STAMP}")" diff --git a/scripts/install/python.sh b/scripts/install/python.sh index 926c5778d3..d7a3b1de20 100755 --- a/scripts/install/python.sh +++ b/scripts/install/python.sh @@ -3,6 +3,9 @@ set -euo pipefail : "${PYTHON:?PYTHON not set}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}/../.." + EXTRA="${1:-}" retry() { diff --git a/scripts/tests/deploy/ci/all.sh b/scripts/tests/deploy/ci/all.sh index bddf1e10d8..11bc532a05 100755 --- a/scripts/tests/deploy/ci/all.sh +++ b/scripts/tests/deploy/ci/all.sh @@ -37,6 +37,6 @@ fi export INFINITO_INVENTORY_DIR MISSING_ONLY echo ">>> Installing CI dependencies" -"${PYTHON}" -m pip install ruamel.yaml PyYAML humanfriendly +PYTHON="${PYTHON}" bash "${REPO_ROOT}/scripts/install/python.sh" exec "${REPO_ROOT}/scripts/tests/deploy/distros.sh" "${SCRIPT_DIR}/one.sh" diff --git a/tests/lint/repository/dependencies/test_ci_host_dependencies.py b/tests/lint/repository/dependencies/test_ci_host_dependencies.py new file mode 100644 index 0000000000..8607eff673 --- /dev/null +++ b/tests/lint/repository/dependencies/test_ci_host_dependencies.py @@ -0,0 +1,131 @@ +"""Lint guard: the runner-host CLI imports against the base dependency set only. + +`scripts/tests/deploy/ci/all.sh` bootstraps the GitHub Actions runner host with +`scripts/install/python.sh`, which installs the project without extras. The host +then runs `cli.administration.deploy.development` outside the project image, so +every module-level import that chain reaches must be satisfiable from +`[project.dependencies]` alone. A package that only `[project.optional-dependencies].deploy` +declares is present in the image, not on the host. + +The check imports the CLI's command modules with every `deploy` package blocked, +which is what a host that installed the base set sees. Before this guard the gap +was invisible: the `ubuntu-latest` image happened to preinstall the missing +packages, so a new import surfaced only when GitHub rebased the image or the +import reached a package the image never shipped. + +Optionally-imported packages stay invisible here on purpose: `utils.cache.base` +wraps its ansible import in `try/except`, so blocking ansible changes nothing and +ansible correctly never has to reach the host. +""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +import tomllib +import unittest + +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +_PYPROJECT = PROJECT_ROOT / "pyproject.toml" + +_DEFAULT_ENV = PROJECT_ROOT / "default.env" + +_SRC_DIR_RE = re.compile(r"^INFINITO_SRC_DIR=(.+)$", re.MULTILINE) + +_DEPLOY_EXTRA = "deploy" + +_PROBE = """ +import sys + +blocked = set(sys.argv[1].split(",")) + + +class ImageOnlyBlocker: + def find_spec(self, fullname, path=None, target=None): + top = fullname.split(".")[0] + if top in blocked: + raise ImportError( + f"{top!r} is a [project.optional-dependencies].deploy package " + "and is not installed on the runner host" + ) + return None + + +sys.meta_path.insert(0, ImageOnlyBlocker()) + +from cli.administration.deploy.development.cli import _build_parser + +_build_parser("cli.administration.deploy.development") +""" + + +def _top_level_name(requirement: str) -> str: + """Return the import name a requirement string installs. + + Args: + requirement: a PEP 508 requirement as written in `pyproject.toml`, + optionally carrying a direct-URL (`name @ https://...`) or a + version specifier. + """ + name = re.split(r"[\s@<>=!~\[;]", requirement.strip(), maxsplit=1)[0] + return name.replace("-", "_") + + +def _deploy_top_level_names() -> list[str]: + data = tomllib.loads(read_text(str(_PYPROJECT))) + extras = data["project"]["optional-dependencies"] + if _DEPLOY_EXTRA not in extras: + raise AssertionError( + f"pyproject.toml declares no '{_DEPLOY_EXTRA}' extra; the runner host " + "bootstrap has no way to tell base from image-only dependencies" + ) + return sorted({_top_level_name(item) for item in extras[_DEPLOY_EXTRA]}) + + +def _src_dir() -> str: + match = _SRC_DIR_RE.search(read_text(str(_DEFAULT_ENV))) + if match is None: + raise AssertionError(f"INFINITO_SRC_DIR is not declared in {_DEFAULT_ENV}") + return match.group(1).strip() + + +class TestCiHostDependencies(unittest.TestCase): + """The runner-host CLI builds its parser without any `deploy` package.""" + + def test_cli_parser_builds_without_deploy_extra(self) -> None: + blocked = _deploy_top_level_names() + + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + path for path in (str(PROJECT_ROOT), env.get("PYTHONPATH", "")) if path + ) + env["INFINITO_SRC_DIR"] = _src_dir() + + result = subprocess.run( + [sys.executable, "-c", _PROBE, ",".join(blocked)], + cwd=str(PROJECT_ROOT), + env=env, + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual( + result.returncode, + 0, + msg=( + "cli.administration.deploy.development imports a package that the " + "runner host does not install. Either move it into " + "[project.dependencies] (it is then paid for on every CI job) or " + "make the import lazy:\n" + result.stderr + ), + ) + + +if __name__ == "__main__": # pragma: no cover + unittest.main()