diff --git a/roles/dev-python/files/shell/install.sh b/roles/dev-python/files/shell/install.sh index 961b8d8b7c..3b524042ed 100755 --- a/roles/dev-python/files/shell/install.sh +++ b/roles/dev-python/files/shell/install.sh @@ -1,28 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -MODE="${1:-ensure}" -case "${MODE}" in - ensure | print) ;; - *) - echo "Usage: $0 [ensure|print]" >&2 - exit 2 - ;; -esac +PYTHON_MIN_MINOR=11 +PYTHON_MAX_MINOR=14 -if [[ "${MODE}" == "print" ]]; then - exec 3>&1 - exec 1>&2 -fi - -if [[ ! -f /etc/os-release ]]; then - echo "[ERROR] /etc/os-release not found; unsupported system." >&2 - exit 1 -fi - -# shellcheck disable=SC1091 -. /etc/os-release -ID_LIKE="${ID_LIKE:-}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" DOCKER_APT_SANITIZER="${REPO_ROOT}/roles/sys-svc-container/files/shell/apt_repo.sh" @@ -67,9 +48,25 @@ sanitize_debian_docker_apt_sources() { run_privileged bash "${DOCKER_APT_SANITIZER}" sanitize-sources "${distro_id}" 1 >/dev/null } +python_minor_within_bounds() { + local version="$1" + [[ "${version}" =~ ^3\.([0-9]+)$ ]] || return 1 + (( BASH_REMATCH[1] >= PYTHON_MIN_MINOR && BASH_REMATCH[1] <= PYTHON_MAX_MINOR )) +} + +keep_python_pkgs_within_bounds() { + local pkg + while IFS= read -r pkg; do + if python_minor_within_bounds "${pkg#python}"; then + printf '%s\n' "${pkg}" + fi + done +} + find_highest_apt_python_pkg() { apt-cache pkgnames 2>/dev/null \ | grep -E '^python3\.[0-9]+$' \ + | keep_python_pkgs_within_bounds \ | sort -V \ | tail -n1 } @@ -81,6 +78,7 @@ find_highest_rpm_python_pkg() { run_privileged "${pm}" -q list --available "python3.[0-9]*" 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error } \ | awk '/^python3\.[0-9]+\./ {pkg=$1; sub(/\.[^.]+$/, "", pkg); print pkg}' \ + | keep_python_pkgs_within_bounds \ | sort -Vu \ | tail -n1 } @@ -124,7 +122,7 @@ install_python_packages() { run_privileged apt-get update run_privileged apt-get install -y --no-install-recommends python3 python3-pip apt_best_pkg="$(find_highest_apt_python_pkg || true)" - if [[ "${apt_best_pkg}" =~ ^python3\.([0-9]+)$ ]] && (( BASH_REMATCH[1] >= 11 )); then + if [[ -n "${apt_best_pkg}" ]]; then run_privileged apt-get install -y --no-install-recommends "${apt_best_pkg}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error if apt-cache show "${apt_best_pkg}-venv" >/dev/null 2>&1; then run_privileged apt-get install -y --no-install-recommends "${apt_best_pkg}-venv" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error @@ -138,7 +136,7 @@ install_python_packages() { run_privileged dnf -y install python3 python3-pip dnf_best_pkg="$(find_highest_rpm_python_pkg dnf || true)" - if [[ "${dnf_best_pkg}" =~ ^python3\.([0-9]+)$ ]] && (( BASH_REMATCH[1] >= 11 )); then + if [[ -n "${dnf_best_pkg}" ]]; then install_versioned_rpm_python_and_pip dnf "${dnf_best_pkg}" fi run_privileged dnf -y clean all || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error @@ -155,7 +153,7 @@ install_python_packages() { fi run_privileged "${PM}" -y install python3 python3-pip || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error rpm_best_pkg="$(find_highest_rpm_python_pkg "${PM}" || true)" - if [[ "${rpm_best_pkg}" =~ ^python3\.([0-9]+)$ ]] && (( BASH_REMATCH[1] >= 11 )); then + if [[ -n "${rpm_best_pkg}" ]]; then install_versioned_rpm_python_and_pip "${PM}" "${rpm_best_pkg}" fi run_privileged "${PM}" -y clean all || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error @@ -173,7 +171,7 @@ install_python_packages() { fi run_privileged "${PM}" -y install python3 python3-pip || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error rpm_like_best_pkg="$(find_highest_rpm_python_pkg "${PM}" || true)" - if [[ "${rpm_like_best_pkg}" =~ ^python3\.([0-9]+)$ ]] && (( BASH_REMATCH[1] >= 11 )); then + if [[ -n "${rpm_like_best_pkg}" ]]; then install_versioned_rpm_python_and_pip "${PM}" "${rpm_like_best_pkg}" fi run_privileged "${PM}" -y clean all || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error @@ -186,15 +184,6 @@ install_python_packages() { esac } -python_is_311_or_higher() { - local bin="$1" - command -v "${bin}" >/dev/null 2>&1 || return 1 - "${bin}" - <<'PY' >/dev/null 2>&1 -import sys -raise SystemExit(0 if sys.version_info >= (3, 11) else 1) -PY -} - python_has_pip_module() { local bin="$1" command -v "${bin}" >/dev/null 2>&1 || return 1 @@ -221,16 +210,11 @@ canonicalize_python_candidate() { printf '%s\n' "${bin}" } -pick_python_bin() { +pick_best_python() { local bin candidate name version local best_version="" best_bin="" local best_pip_version="" best_pip_bin="" - for bin in \ - /usr/local/bin/python3.* \ - /usr/bin/python3.* \ - /usr/local/bin/python3 \ - /usr/bin/python3 \ - python3; do + for bin in "$@"; do [[ -x "${bin}" ]] || continue candidate="$(canonicalize_python_candidate "${bin}")" [[ -x "${candidate}" ]] || continue @@ -238,11 +222,8 @@ pick_python_bin() { if [[ "${name}" == python3.* ]] && [[ ! "${name}" =~ ^python3\.[0-9]+$ ]]; then continue fi - if ! python_is_311_or_higher "${candidate}"; then - continue - fi - version="$(python_minor_version "${candidate}")" - [[ -n "${version}" ]] || continue + version="$(python_minor_version "${candidate}")" || continue + python_minor_within_bounds "${version}" || continue if [[ -z "${best_version}" ]] || [[ "$(printf '%s\n%s\n' "${best_version}" "${version}" | sort -V | tail -n1)" == "${version}" ]]; then best_version="${version}" best_bin="${candidate}" @@ -262,6 +243,15 @@ pick_python_bin() { printf '%s\n' "${best_bin}" } +pick_python_bin() { + pick_best_python \ + /usr/local/bin/python3.* \ + /usr/bin/python3.* \ + /usr/local/bin/python3 \ + /usr/bin/python3 \ + python3 +} + configure_defaults() { local pybin="$1" local pyver @@ -315,7 +305,7 @@ resolve_python_bin_or_fail() { if PYBIN="$(pick_python_bin)"; then return 0 fi - echo "[ERROR] No Python >= 3.11 found after installation attempt." >&2 + echo "[ERROR] No Python 3.${PYTHON_MIN_MINOR} to 3.${PYTHON_MAX_MINOR} found after installation attempt; newer interpreters are rejected." >&2 if command -v python3 >/dev/null 2>&1; then python3 --version || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error fi @@ -323,7 +313,7 @@ resolve_python_bin_or_fail() { } ensure_python_bin() { - log "Ensuring highest available Python >=3.11 for ID=${ID} ID_LIKE=${ID_LIKE}" + log "Ensuring highest available Python 3.${PYTHON_MIN_MINOR} to 3.${PYTHON_MAX_MINOR} for ID=${ID} ID_LIKE=${ID_LIKE}" install_python_packages hash -r || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error resolve_python_bin_or_fail @@ -336,23 +326,52 @@ resolve_or_install_python_bin() { ensure_python_bin } -if [[ "${MODE}" == "ensure" ]]; then - if PYBIN="$(pick_python_bin)"; then - if ! can_run_privileged; then - log "Using existing Python without privileged system-wide symlink updates: $("${PYBIN}" --version)" - if "${PYBIN}" -m pip --version >/dev/null 2>&1; then - log "Existing pip: $("${PYBIN}" -m pip --version)" - exit 0 - fi - fi - else - ensure_python_bin +main() { + MODE="${1:-ensure}" + case "${MODE}" in + ensure | print) ;; + *) + echo "Usage: $0 [ensure|print]" >&2 + exit 2 + ;; + esac + + if [[ "${MODE}" == "print" ]]; then + exec 3>&1 + exec 1>&2 fi - configure_defaults "${PYBIN}" - log "Default python: $(python3 --version)" - log "Default pip: $(pip3 --version)" -else - resolve_or_install_python_bin - printf '%s\n' "${PYBIN}" >&3 + if [[ ! -f /etc/os-release ]]; then + echo "[ERROR] /etc/os-release not found; unsupported system." >&2 + exit 1 + fi + + # shellcheck disable=SC1091 + . /etc/os-release + ID_LIKE="${ID_LIKE:-}" + + if [[ "${MODE}" == "ensure" ]]; then + if PYBIN="$(pick_python_bin)"; then + if ! can_run_privileged; then + log "Using existing Python without privileged system-wide symlink updates: $("${PYBIN}" --version)" + if "${PYBIN}" -m pip --version >/dev/null 2>&1; then + log "Existing pip: $("${PYBIN}" -m pip --version)" + exit 0 + fi + fi + else + ensure_python_bin + fi + + configure_defaults "${PYBIN}" + log "Default python: $(python3 --version)" + log "Default pip: $(pip3 --version)" + else + resolve_or_install_python_bin + printf '%s\n' "${PYBIN}" >&3 + fi +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" fi diff --git a/tests/unit/python/roles/dev-python/__init__.py b/tests/unit/python/roles/dev-python/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/tests/unit/python/roles/dev-python/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/unit/python/roles/dev-python/files/__init__.py b/tests/unit/python/roles/dev-python/files/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/tests/unit/python/roles/dev-python/files/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/unit/python/roles/dev-python/files/test_install.py b/tests/unit/python/roles/dev-python/files/test_install.py new file mode 100644 index 0000000000..e681cd73a5 --- /dev/null +++ b/tests/unit/python/roles/dev-python/files/test_install.py @@ -0,0 +1,166 @@ +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "roles/dev-python/files/shell/install.sh" + + +def _call(snippet: str, *args: str) -> subprocess.CompletedProcess: + """Source install.sh without running its entrypoint, then run a snippet. + + Args: + snippet: bash code evaluated after sourcing; sees ``args`` as ``"$@"``. + args: positional arguments handed to the snippet. + + Returns: + The completed bash process. + """ + return subprocess.run( + ["bash", "-c", f'source "$1"; shift; {snippet}', "bash", str(SCRIPT), *args], + capture_output=True, + text=True, + check=False, + ) + + +def _bound(name: str) -> int: + return int(_call(f'printf "%s" "${name}"').stdout) + + +def _fake_python(directory: Path, minor: int, *, pip: bool) -> str: + """Write an executable that answers the interpreter probes of install.sh. + + Args: + directory: directory the fake is written to. + minor: Python 3 minor version the fake reports. + pip: whether ``-m pip --version`` succeeds. + + Returns: + Absolute path of the fake interpreter. + """ + path = directory / f"python3.{minor}" + path.write_text( + "#!/bin/sh\n" + f'if [ "$1" = "-m" ]; then exit {0 if pip else 1}; fi\n' + "cat >/dev/null\n" + f"echo 3.{minor}\n" + ) + path.chmod(0o755) + return str(path) + + +class _InterpreterCase(unittest.TestCase): + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = Path(tmp.name) + self.floor = _bound("PYTHON_MIN_MINOR") + self.cap = _bound("PYTHON_MAX_MINOR") + + def fake(self, minor: int, *, pip: bool = True) -> str: + return _fake_python(self.dir, minor, pip=pip) + + +class TestPickBestPython(_InterpreterCase): + def pick(self, *candidates: str) -> subprocess.CompletedProcess: + return _call('pick_best_python "$@"', *candidates) + + def test_skips_a_minor_above_the_cap(self) -> None: + capped = self.fake(self.cap) + result = self.pick(capped, self.fake(self.cap + 1)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), capped) + + def test_ignores_pip_on_a_minor_above_the_cap(self) -> None: + capped = self.fake(self.cap, pip=False) + result = self.pick( + self.fake(self.cap - 1, pip=False), capped, self.fake(self.cap + 1) + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), capped) + + def test_picks_the_highest_minor_within_bounds(self) -> None: + highest = self.fake(self.cap) + result = self.pick(self.fake(self.cap - 2), self.fake(self.cap - 1), highest) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), highest) + + def test_prefers_a_lower_minor_that_has_pip(self) -> None: + with_pip = self.fake(self.cap - 1) + result = self.pick(with_pip, self.fake(self.cap, pip=False)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), with_pip) + + def test_fails_when_only_minors_above_the_cap_exist(self) -> None: + result = self.pick(self.fake(self.cap + 1)) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + + def test_fails_when_only_minors_below_the_floor_exist(self) -> None: + result = self.pick(self.fake(self.floor - 1)) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + + +class TestResolvePythonBinOrFail(_InterpreterCase): + def test_names_the_supported_range_when_only_a_newer_minor_exists(self) -> None: + result = _call( + 'candidates=("$@"); ' + 'pick_python_bin() { pick_best_python "${candidates[@]}"; }; ' + "resolve_python_bin_or_fail", + self.fake(self.cap + 1), + ) + self.assertEqual(result.returncode, 1) + self.assertIn(f"3.{self.floor} to 3.{self.cap}", result.stderr) + + +class TestFindHighestPythonPackage(unittest.TestCase): + def setUp(self) -> None: + self.cap = _bound("PYTHON_MAX_MINOR") + + def rpm(self, *minors: int) -> subprocess.CompletedProcess: + listing = "Available packages\n" + "".join( + f"python3.{m}.x86_64 3.{m}.0-1.fc45 fedora\n" + f"python3.{m}-devel.x86_64 3.{m}.0-1.fc45 fedora\n" + for m in minors + ) + return _call( + 'listing="$1"; ' + 'run_privileged() { "$@"; }; ' + 'dnf() { printf "%s" "${listing}"; }; ' + "find_highest_rpm_python_pkg dnf", + listing, + ) + + def apt(self, *minors: int) -> subprocess.CompletedProcess: + names = ["python3", "python3-pip"] + [f"python3.{m}" for m in minors] + return _call( + 'names=("$@"); ' + 'apt-cache() { printf "%s\\n" "${names[@]}"; }; ' + "find_highest_apt_python_pkg", + *names, + ) + + def test_rpm_skips_a_package_above_the_cap(self) -> None: + result = self.rpm(self.cap - 1, self.cap, self.cap + 1) + self.assertEqual(result.stdout.strip(), f"python3.{self.cap}") + + def test_rpm_yields_nothing_when_only_packages_above_the_cap_exist(self) -> None: + result = self.rpm(self.cap + 1) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "") + + def test_apt_skips_a_package_above_the_cap(self) -> None: + result = self.apt(self.cap - 1, self.cap, self.cap + 1) + self.assertEqual(result.stdout.strip(), f"python3.{self.cap}") + + def test_apt_keeps_the_highest_package_within_bounds(self) -> None: + result = self.apt(self.cap - 2, self.cap - 1) + self.assertEqual(result.stdout.strip(), f"python3.{self.cap - 1}") + + +if __name__ == "__main__": + unittest.main()