mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 02:54:26 +00:00
fix(dev-python): cap the bootstrap Python at the newest minor ansible-core supports
Fedora 45 defaults to Python 3.15. ansible-core 2.21, which the unpinned `ansible` extra still resolves to (ansible 14.5.0 pins ~=2.21.5), calls dataclasses._is_type, which 3.15 removed, so the Workspace (fedora:latest) job died before the first playbook (run 41491308729). install.sh now picks and installs the highest Python between PYTHON_MIN_MINOR and PYTHON_MAX_MINOR (3.11 to 3.14) on apt, rpm and pick_python_bin, and fails loudly when only newer interpreters exist. On Fedora 45 it installs python3.14. Raise PYTHON_MAX_MINOR once ansible ships ansible-core 2.22. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
kevinveenbirkenbachandClaude Opus 5.5 committed
1 parent
d807d1b6b4
commit
bf9c5be173
4 files changed
+252
-65
No files matched your search
@@ -1,28 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
MODE="${1:-ensure}"
|
||||
case "${MODE}" in
|
||||
ensure | print) ;;
|
||||
*)
|
||||
echo "Usage: $0 [ensure|print]" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
PYTHON_MIN_MINOR=11
|
||||
PYTHON_MAX_MINOR=14
|
||||
|
||||
if [[ "${MODE}" == "print" ]]; then
|
||||
exec 3>&1
|
||||
exec 1>&2
|
||||
fi
|
||||
|
||||
if [[ ! -f /etc/os-release ]]; then
|
||||
echo "[ERROR] /etc/os-release not found; unsupported system." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
. /etc/os-release
|
||||
ID_LIKE="${ID_LIKE:-}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
|
||||
DOCKER_APT_SANITIZER="${REPO_ROOT}/roles/sys-svc-container/files/shell/apt_repo.sh"
|
||||
@@ -67,9 +48,25 @@ sanitize_debian_docker_apt_sources() {
|
||||
run_privileged bash "${DOCKER_APT_SANITIZER}" sanitize-sources "${distro_id}" 1 >/dev/null
|
||||
}
|
||||
|
||||
python_minor_within_bounds() {
|
||||
local version="$1"
|
||||
[[ "${version}" =~ ^3\.([0-9]+)$ ]] || return 1
|
||||
(( BASH_REMATCH[1] >= PYTHON_MIN_MINOR && BASH_REMATCH[1] <= PYTHON_MAX_MINOR ))
|
||||
}
|
||||
|
||||
keep_python_pkgs_within_bounds() {
|
||||
local pkg
|
||||
while IFS= read -r pkg; do
|
||||
if python_minor_within_bounds "${pkg#python}"; then
|
||||
printf '%s\n' "${pkg}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
find_highest_apt_python_pkg() {
|
||||
apt-cache pkgnames 2>/dev/null \
|
||||
| grep -E '^python3\.[0-9]+$' \
|
||||
| keep_python_pkgs_within_bounds \
|
||||
| sort -V \
|
||||
| tail -n1
|
||||
}
|
||||
@@ -81,6 +78,7 @@ find_highest_rpm_python_pkg() {
|
||||
run_privileged "${pm}" -q list --available "python3.[0-9]*" 2>/dev/null || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
||||
} \
|
||||
| awk '/^python3\.[0-9]+\./ {pkg=$1; sub(/\.[^.]+$/, "", pkg); print pkg}' \
|
||||
| keep_python_pkgs_within_bounds \
|
||||
| sort -Vu \
|
||||
| tail -n1
|
||||
}
|
||||
@@ -124,7 +122,7 @@ install_python_packages() {
|
||||
run_privileged apt-get update
|
||||
run_privileged apt-get install -y --no-install-recommends python3 python3-pip
|
||||
apt_best_pkg="$(find_highest_apt_python_pkg || true)"
|
||||
if [[ "${apt_best_pkg}" =~ ^python3\.([0-9]+)$ ]] && (( BASH_REMATCH[1] >= 11 )); then
|
||||
if [[ -n "${apt_best_pkg}" ]]; then
|
||||
run_privileged apt-get install -y --no-install-recommends "${apt_best_pkg}" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
||||
if apt-cache show "${apt_best_pkg}-venv" >/dev/null 2>&1; then
|
||||
run_privileged apt-get install -y --no-install-recommends "${apt_best_pkg}-venv" || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
||||
@@ -138,7 +136,7 @@ install_python_packages() {
|
||||
|
||||
run_privileged dnf -y install python3 python3-pip
|
||||
dnf_best_pkg="$(find_highest_rpm_python_pkg dnf || true)"
|
||||
if [[ "${dnf_best_pkg}" =~ ^python3\.([0-9]+)$ ]] && (( BASH_REMATCH[1] >= 11 )); then
|
||||
if [[ -n "${dnf_best_pkg}" ]]; then
|
||||
install_versioned_rpm_python_and_pip dnf "${dnf_best_pkg}"
|
||||
fi
|
||||
run_privileged dnf -y clean all || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
||||
@@ -155,7 +153,7 @@ install_python_packages() {
|
||||
fi
|
||||
run_privileged "${PM}" -y install python3 python3-pip || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
||||
rpm_best_pkg="$(find_highest_rpm_python_pkg "${PM}" || true)"
|
||||
if [[ "${rpm_best_pkg}" =~ ^python3\.([0-9]+)$ ]] && (( BASH_REMATCH[1] >= 11 )); then
|
||||
if [[ -n "${rpm_best_pkg}" ]]; then
|
||||
install_versioned_rpm_python_and_pip "${PM}" "${rpm_best_pkg}"
|
||||
fi
|
||||
run_privileged "${PM}" -y clean all || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
||||
@@ -173,7 +171,7 @@ install_python_packages() {
|
||||
fi
|
||||
run_privileged "${PM}" -y install python3 python3-pip || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
||||
rpm_like_best_pkg="$(find_highest_rpm_python_pkg "${PM}" || true)"
|
||||
if [[ "${rpm_like_best_pkg}" =~ ^python3\.([0-9]+)$ ]] && (( BASH_REMATCH[1] >= 11 )); then
|
||||
if [[ -n "${rpm_like_best_pkg}" ]]; then
|
||||
install_versioned_rpm_python_and_pip "${PM}" "${rpm_like_best_pkg}"
|
||||
fi
|
||||
run_privileged "${PM}" -y clean all || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
||||
@@ -186,15 +184,6 @@ install_python_packages() {
|
||||
esac
|
||||
}
|
||||
|
||||
python_is_311_or_higher() {
|
||||
local bin="$1"
|
||||
command -v "${bin}" >/dev/null 2>&1 || return 1
|
||||
"${bin}" - <<'PY' >/dev/null 2>&1
|
||||
import sys
|
||||
raise SystemExit(0 if sys.version_info >= (3, 11) else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
python_has_pip_module() {
|
||||
local bin="$1"
|
||||
command -v "${bin}" >/dev/null 2>&1 || return 1
|
||||
@@ -221,16 +210,11 @@ canonicalize_python_candidate() {
|
||||
printf '%s\n' "${bin}"
|
||||
}
|
||||
|
||||
pick_python_bin() {
|
||||
pick_best_python() {
|
||||
local bin candidate name version
|
||||
local best_version="" best_bin=""
|
||||
local best_pip_version="" best_pip_bin=""
|
||||
for bin in \
|
||||
/usr/local/bin/python3.* \
|
||||
/usr/bin/python3.* \
|
||||
/usr/local/bin/python3 \
|
||||
/usr/bin/python3 \
|
||||
python3; do
|
||||
for bin in "$@"; do
|
||||
[[ -x "${bin}" ]] || continue
|
||||
candidate="$(canonicalize_python_candidate "${bin}")"
|
||||
[[ -x "${candidate}" ]] || continue
|
||||
@@ -238,11 +222,8 @@ pick_python_bin() {
|
||||
if [[ "${name}" == python3.* ]] && [[ ! "${name}" =~ ^python3\.[0-9]+$ ]]; then
|
||||
continue
|
||||
fi
|
||||
if ! python_is_311_or_higher "${candidate}"; then
|
||||
continue
|
||||
fi
|
||||
version="$(python_minor_version "${candidate}")"
|
||||
[[ -n "${version}" ]] || continue
|
||||
version="$(python_minor_version "${candidate}")" || continue
|
||||
python_minor_within_bounds "${version}" || continue
|
||||
if [[ -z "${best_version}" ]] || [[ "$(printf '%s\n%s\n' "${best_version}" "${version}" | sort -V | tail -n1)" == "${version}" ]]; then
|
||||
best_version="${version}"
|
||||
best_bin="${candidate}"
|
||||
@@ -262,6 +243,15 @@ pick_python_bin() {
|
||||
printf '%s\n' "${best_bin}"
|
||||
}
|
||||
|
||||
pick_python_bin() {
|
||||
pick_best_python \
|
||||
/usr/local/bin/python3.* \
|
||||
/usr/bin/python3.* \
|
||||
/usr/local/bin/python3 \
|
||||
/usr/bin/python3 \
|
||||
python3
|
||||
}
|
||||
|
||||
configure_defaults() {
|
||||
local pybin="$1"
|
||||
local pyver
|
||||
@@ -315,7 +305,7 @@ resolve_python_bin_or_fail() {
|
||||
if PYBIN="$(pick_python_bin)"; then
|
||||
return 0
|
||||
fi
|
||||
echo "[ERROR] No Python >= 3.11 found after installation attempt." >&2
|
||||
echo "[ERROR] No Python 3.${PYTHON_MIN_MINOR} to 3.${PYTHON_MAX_MINOR} found after installation attempt; newer interpreters are rejected." >&2
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
python3 --version || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
||||
fi
|
||||
@@ -323,7 +313,7 @@ resolve_python_bin_or_fail() {
|
||||
}
|
||||
|
||||
ensure_python_bin() {
|
||||
log "Ensuring highest available Python >=3.11 for ID=${ID} ID_LIKE=${ID_LIKE}"
|
||||
log "Ensuring highest available Python 3.${PYTHON_MIN_MINOR} to 3.${PYTHON_MAX_MINOR} for ID=${ID} ID_LIKE=${ID_LIKE}"
|
||||
install_python_packages
|
||||
hash -r || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
|
||||
resolve_python_bin_or_fail
|
||||
@@ -336,23 +326,52 @@ resolve_or_install_python_bin() {
|
||||
ensure_python_bin
|
||||
}
|
||||
|
||||
if [[ "${MODE}" == "ensure" ]]; then
|
||||
if PYBIN="$(pick_python_bin)"; then
|
||||
if ! can_run_privileged; then
|
||||
log "Using existing Python without privileged system-wide symlink updates: $("${PYBIN}" --version)"
|
||||
if "${PYBIN}" -m pip --version >/dev/null 2>&1; then
|
||||
log "Existing pip: $("${PYBIN}" -m pip --version)"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
else
|
||||
ensure_python_bin
|
||||
main() {
|
||||
MODE="${1:-ensure}"
|
||||
case "${MODE}" in
|
||||
ensure | print) ;;
|
||||
*)
|
||||
echo "Usage: $0 [ensure|print]" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "${MODE}" == "print" ]]; then
|
||||
exec 3>&1
|
||||
exec 1>&2
|
||||
fi
|
||||
|
||||
configure_defaults "${PYBIN}"
|
||||
log "Default python: $(python3 --version)"
|
||||
log "Default pip: $(pip3 --version)"
|
||||
else
|
||||
resolve_or_install_python_bin
|
||||
printf '%s\n' "${PYBIN}" >&3
|
||||
if [[ ! -f /etc/os-release ]]; then
|
||||
echo "[ERROR] /etc/os-release not found; unsupported system." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
. /etc/os-release
|
||||
ID_LIKE="${ID_LIKE:-}"
|
||||
|
||||
if [[ "${MODE}" == "ensure" ]]; then
|
||||
if PYBIN="$(pick_python_bin)"; then
|
||||
if ! can_run_privileged; then
|
||||
log "Using existing Python without privileged system-wide symlink updates: $("${PYBIN}" --version)"
|
||||
if "${PYBIN}" -m pip --version >/dev/null 2>&1; then
|
||||
log "Existing pip: $("${PYBIN}" -m pip --version)"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
else
|
||||
ensure_python_bin
|
||||
fi
|
||||
|
||||
configure_defaults "${PYBIN}"
|
||||
log "Default python: $(python3 --version)"
|
||||
log "Default pip: $(pip3 --version)"
|
||||
else
|
||||
resolve_or_install_python_bin
|
||||
printf '%s\n' "${PYBIN}" >&3
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
main "$@"
|
||||
fi
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from utils.cache.files import PROJECT_ROOT
|
||||
|
||||
SCRIPT = PROJECT_ROOT / "roles/dev-python/files/shell/install.sh"
|
||||
|
||||
|
||||
def _call(snippet: str, *args: str) -> subprocess.CompletedProcess:
|
||||
"""Source install.sh without running its entrypoint, then run a snippet.
|
||||
|
||||
Args:
|
||||
snippet: bash code evaluated after sourcing; sees ``args`` as ``"$@"``.
|
||||
args: positional arguments handed to the snippet.
|
||||
|
||||
Returns:
|
||||
The completed bash process.
|
||||
"""
|
||||
return subprocess.run(
|
||||
["bash", "-c", f'source "$1"; shift; {snippet}', "bash", str(SCRIPT), *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def _bound(name: str) -> int:
|
||||
return int(_call(f'printf "%s" "${name}"').stdout)
|
||||
|
||||
|
||||
def _fake_python(directory: Path, minor: int, *, pip: bool) -> str:
|
||||
"""Write an executable that answers the interpreter probes of install.sh.
|
||||
|
||||
Args:
|
||||
directory: directory the fake is written to.
|
||||
minor: Python 3 minor version the fake reports.
|
||||
pip: whether ``-m pip --version`` succeeds.
|
||||
|
||||
Returns:
|
||||
Absolute path of the fake interpreter.
|
||||
"""
|
||||
path = directory / f"python3.{minor}"
|
||||
path.write_text(
|
||||
"#!/bin/sh\n"
|
||||
f'if [ "$1" = "-m" ]; then exit {0 if pip else 1}; fi\n'
|
||||
"cat >/dev/null\n"
|
||||
f"echo 3.{minor}\n"
|
||||
)
|
||||
path.chmod(0o755)
|
||||
return str(path)
|
||||
|
||||
|
||||
class _InterpreterCase(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.dir = Path(tmp.name)
|
||||
self.floor = _bound("PYTHON_MIN_MINOR")
|
||||
self.cap = _bound("PYTHON_MAX_MINOR")
|
||||
|
||||
def fake(self, minor: int, *, pip: bool = True) -> str:
|
||||
return _fake_python(self.dir, minor, pip=pip)
|
||||
|
||||
|
||||
class TestPickBestPython(_InterpreterCase):
|
||||
def pick(self, *candidates: str) -> subprocess.CompletedProcess:
|
||||
return _call('pick_best_python "$@"', *candidates)
|
||||
|
||||
def test_skips_a_minor_above_the_cap(self) -> None:
|
||||
capped = self.fake(self.cap)
|
||||
result = self.pick(capped, self.fake(self.cap + 1))
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.strip(), capped)
|
||||
|
||||
def test_ignores_pip_on_a_minor_above_the_cap(self) -> None:
|
||||
capped = self.fake(self.cap, pip=False)
|
||||
result = self.pick(
|
||||
self.fake(self.cap - 1, pip=False), capped, self.fake(self.cap + 1)
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.strip(), capped)
|
||||
|
||||
def test_picks_the_highest_minor_within_bounds(self) -> None:
|
||||
highest = self.fake(self.cap)
|
||||
result = self.pick(self.fake(self.cap - 2), self.fake(self.cap - 1), highest)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.strip(), highest)
|
||||
|
||||
def test_prefers_a_lower_minor_that_has_pip(self) -> None:
|
||||
with_pip = self.fake(self.cap - 1)
|
||||
result = self.pick(with_pip, self.fake(self.cap, pip=False))
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.strip(), with_pip)
|
||||
|
||||
def test_fails_when_only_minors_above_the_cap_exist(self) -> None:
|
||||
result = self.pick(self.fake(self.cap + 1))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, "")
|
||||
|
||||
def test_fails_when_only_minors_below_the_floor_exist(self) -> None:
|
||||
result = self.pick(self.fake(self.floor - 1))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, "")
|
||||
|
||||
|
||||
class TestResolvePythonBinOrFail(_InterpreterCase):
|
||||
def test_names_the_supported_range_when_only_a_newer_minor_exists(self) -> None:
|
||||
result = _call(
|
||||
'candidates=("$@"); '
|
||||
'pick_python_bin() { pick_best_python "${candidates[@]}"; }; '
|
||||
"resolve_python_bin_or_fail",
|
||||
self.fake(self.cap + 1),
|
||||
)
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn(f"3.{self.floor} to 3.{self.cap}", result.stderr)
|
||||
|
||||
|
||||
class TestFindHighestPythonPackage(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.cap = _bound("PYTHON_MAX_MINOR")
|
||||
|
||||
def rpm(self, *minors: int) -> subprocess.CompletedProcess:
|
||||
listing = "Available packages\n" + "".join(
|
||||
f"python3.{m}.x86_64 3.{m}.0-1.fc45 fedora\n"
|
||||
f"python3.{m}-devel.x86_64 3.{m}.0-1.fc45 fedora\n"
|
||||
for m in minors
|
||||
)
|
||||
return _call(
|
||||
'listing="$1"; '
|
||||
'run_privileged() { "$@"; }; '
|
||||
'dnf() { printf "%s" "${listing}"; }; '
|
||||
"find_highest_rpm_python_pkg dnf",
|
||||
listing,
|
||||
)
|
||||
|
||||
def apt(self, *minors: int) -> subprocess.CompletedProcess:
|
||||
names = ["python3", "python3-pip"] + [f"python3.{m}" for m in minors]
|
||||
return _call(
|
||||
'names=("$@"); '
|
||||
'apt-cache() { printf "%s\\n" "${names[@]}"; }; '
|
||||
"find_highest_apt_python_pkg",
|
||||
*names,
|
||||
)
|
||||
|
||||
def test_rpm_skips_a_package_above_the_cap(self) -> None:
|
||||
result = self.rpm(self.cap - 1, self.cap, self.cap + 1)
|
||||
self.assertEqual(result.stdout.strip(), f"python3.{self.cap}")
|
||||
|
||||
def test_rpm_yields_nothing_when_only_packages_above_the_cap_exist(self) -> None:
|
||||
result = self.rpm(self.cap + 1)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout, "")
|
||||
|
||||
def test_apt_skips_a_package_above_the_cap(self) -> None:
|
||||
result = self.apt(self.cap - 1, self.cap, self.cap + 1)
|
||||
self.assertEqual(result.stdout.strip(), f"python3.{self.cap}")
|
||||
|
||||
def test_apt_keeps_the_highest_package_within_bounds(self) -> None:
|
||||
result = self.apt(self.cap - 2, self.cap - 1)
|
||||
self.assertEqual(result.stdout.strip(), f"python3.{self.cap - 1}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user