diff --git a/scripts/tests/deploy/swarm/utils/_context.sh b/scripts/tests/deploy/swarm/utils/_context.sh index ca3ddce992..18279886a9 100755 --- a/scripts/tests/deploy/swarm/utils/_context.sh +++ b/scripts/tests/deploy/swarm/utils/_context.sh @@ -111,7 +111,7 @@ has_swarm_service() { --format '{{.Name}}' "${STACK_NAME}" 2>/dev/null)"; then _names="" fi - printf '%s\n' "${_names}" | grep -qxF "${SERVICE_NAME}" + grep -qxF "${SERVICE_NAME}" <<<"${_names}" } # Prints the swarm service carrying APP_ID's database. A role that pins its own @@ -124,9 +124,9 @@ resolve_db_service() { if ! _names="$(timeout 30 docker exec "${MGR}" docker service ls --format '{{.Name}}' 2>/dev/null)"; then return 0 fi - if printf '%s\n' "${_names}" | grep -qxF "${STACK_NAME}_database"; then + if grep -qxF "${STACK_NAME}_database" <<<"${_names}"; then printf '%s\n' "${STACK_NAME}_database" - elif printf '%s\n' "${_names}" | grep -qxF "${DB_DEP}_${DB_DEP}"; then + elif grep -qxF "${DB_DEP}_${DB_DEP}" <<<"${_names}"; then printf '%s\n' "${DB_DEP}_${DB_DEP}" fi } diff --git a/tests/external/update/pip/test_ansible_python_cap.py b/tests/external/update/pip/test_ansible_python_cap.py new file mode 100644 index 0000000000..af41f9b527 --- /dev/null +++ b/tests/external/update/pip/test_ansible_python_cap.py @@ -0,0 +1,208 @@ +"""Check the Python cap in ``requires-python`` against the newest ``ansible`` release. + +``pyproject.toml`` caps the supported Python (``>=3.11,<3.15``) because the +unpinned ``ansible`` dependency resolves to an ansible-core that cannot run on +the next minor: ansible-core 2.21 calls ``dataclasses._is_type``, which Python +3.15 removed. No updater touches ``requires-python``, so the cap needs a +watcher. + +This is external because the answer lives on PyPI: the newest ``ansible`` +release, the ansible-core range it requires, and the Python classifiers of the +newest ansible-core inside that range. It contacts ``https://pypi.org/pypi`` +only. + +- The test fails when the cap allows a Python minor that this ansible-core + does not support, because the bootstrap would then pick an interpreter the + deploy cannot run on. +- It only warns when ansible-core supports a newer minor than the cap, because + raising the cap needs a test run on every workspace distribution first. +- It warns and passes when PyPI does not answer. +""" + +from __future__ import annotations + +import http.client +import json +import re +import tomllib +import unittest +import urllib.error +import urllib.request + +from packaging.requirements import Requirement +from packaging.version import InvalidVersion, Version + +from utils.annotations.message import warning +from utils.cache.files import read_text + +from . import PROJECT_ROOT + +PYPI = "https://pypi.org/pypi" +TIMEOUT_SECONDS = 15 +USER_AGENT = "infinito-nexus python cap check" +CLASSIFIER = re.compile(r"^Programming Language :: Python :: 3\.(\d+)$") +REQUIRES_PYTHON = re.compile(r"^>=3\.(\d+),<3\.(\d+)$") +PYPROJECT = PROJECT_ROOT / "pyproject.toml" + + +class PyPIUnavailableError(Exception): + """PyPI did not answer with usable JSON.""" + + +def pypi_json(path: str) -> dict: + """Return one PyPI JSON API document. + + Args: + path: the part after ``/pypi/``, e.g. ``ansible/json``. + + Raises: + PyPIUnavailableError: the request failed or returned no JSON. + """ + request = urllib.request.Request( # noqa: S310 - fixed https PyPI origin, path built from literal package names + f"{PYPI}/{path}", headers={"User-Agent": USER_AGENT} + ) + try: + with urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS) as response: # noqa: S310 - fixed https PyPI origin, path built from literal package names + return json.loads(response.read().decode()) + except ( + urllib.error.URLError, + OSError, + http.client.HTTPException, + json.JSONDecodeError, + ) as exc: + raise PyPIUnavailableError(f"{path}: {exc}") from exc + + +def declared_cap() -> int: + """Return the highest Python 3 minor ``requires-python`` admits.""" + spec = tomllib.loads(read_text(str(PYPROJECT)))["project"]["requires-python"] + match = REQUIRES_PYTHON.fullmatch(spec) + if match is None: + raise AssertionError( + f"pyproject.toml requires-python {spec!r} must read >=3.X,<3.Y" + ) + return int(match[2]) - 1 + + +def core_requirement(ansible: dict) -> Requirement: + """Return the ansible-core requirement of one ``ansible`` release document.""" + for raw in ansible["info"].get("requires_dist") or []: + requirement = Requirement(raw) + if requirement.name == "ansible-core" and requirement.marker is None: + return requirement + raise AssertionError( + f"ansible {ansible['info']['version']} declares no ansible-core requirement" + ) + + +def newest_matching(releases: dict, requirement: Requirement) -> Version: + """Return the newest non-yanked final release that satisfies ``requirement``. + + Args: + releases: the ``releases`` map of a PyPI project document. + requirement: the ansible-core requirement of the ``ansible`` release. + """ + candidates = [] + for raw, files in releases.items(): + try: + version = Version(raw) + except InvalidVersion: + continue + if version.is_prerelease or not any(not f.get("yanked") for f in files): + continue + if version in requirement.specifier: + candidates.append(version) + if not candidates: + raise AssertionError(f"no ansible-core release satisfies {requirement}") + return max(candidates) + + +def highest_classified_minor(core: dict) -> int: + """Return the highest Python 3 minor an ansible-core release classifies.""" + minors = [ + int(match[1]) + for classifier in core["info"].get("classifiers") or [] + if (match := CLASSIFIER.match(classifier)) + ] + if not minors: + raise AssertionError( + f"ansible-core {core['info']['version']} lists no Python 3 classifier" + ) + return max(minors) + + +def verdict(cap: int, supported: int) -> str | None: + """Return the failure for a cap above ``supported``, ``None`` otherwise.""" + if cap > supported: + return ( + f"requires-python admits Python 3.{cap}, but the ansible-core the " + f"newest ansible pulls in supports up to 3.{supported}; lower the " + f"upper bound to <3.{supported + 1}" + ) + return None + + +class TestAnsiblePythonCap(unittest.TestCase): + def test_the_cap_matches_what_the_newest_ansible_runs_on(self) -> None: + try: + ansible = pypi_json("ansible/json") + requirement = core_requirement(ansible) + core_version = newest_matching( + pypi_json("ansible-core/json")["releases"], requirement + ) + core = pypi_json(f"ansible-core/{core_version}/json") + except PyPIUnavailableError as exc: + warning(f"PyPI did not answer, cap not checked: {exc}", title="Python cap") + return + cap = declared_cap() + supported = highest_classified_minor(core) + failure = verdict(cap, supported) + self.assertIsNone(failure, failure) + if supported > cap: + warning( + f"ansible {ansible['info']['version']} pulls ansible-core " + f"{core_version}, which supports Python 3.{supported}; " + f"requires-python still stops at 3.{cap}. Raise it to " + f"<3.{supported + 1} after a workspace run on every distribution.", + title="Python cap can be raised", + file="pyproject.toml", + ) + + +class TestVerdict(unittest.TestCase): + def test_a_cap_above_the_supported_minor_fails(self) -> None: + self.assertIn("lower the upper bound to <3.15", verdict(15, 14)) + + def test_a_cap_at_or_below_the_supported_minor_passes(self) -> None: + self.assertEqual((verdict(14, 14), verdict(14, 15)), (None, None)) + + def test_the_newest_satisfying_final_release_is_chosen(self) -> None: + releases = { + "2.21.5": [{"yanked": False}], + "2.21.6": [{"yanked": False}], + "2.21.7": [{"yanked": True}], + "2.22.0": [{"yanked": False}], + "2.21.8rc1": [{"yanked": False}], + } + self.assertEqual( + newest_matching(releases, Requirement("ansible-core~=2.21.5")), + Version("2.21.6"), + ) + + def test_the_highest_python_classifier_wins(self) -> None: + core = { + "info": { + "version": "2.21.6", + "classifiers": [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.13", + ], + } + } + self.assertEqual(highest_classified_minor(core), 14) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/scripts/tests/deploy/swarm/utils/test_resolve_db_service.py b/tests/unit/python/scripts/tests/deploy/swarm/utils/test_resolve_db_service.py index 0eb816192d..9e2a335286 100644 --- a/tests/unit/python/scripts/tests/deploy/swarm/utils/test_resolve_db_service.py +++ b/tests/unit/python/scripts/tests/deploy/swarm/utils/test_resolve_db_service.py @@ -28,10 +28,13 @@ PINNED_ENGINE_APP = "web-app-magento" NO_DB_APP = "web-svc-html" LOCAL_SERVICE = "magento_database" CENTRAL_SERVICE = "mariadb_mariadb" +FILLER = tuple(f"filler_service_{index}" for index in range(10000)) class TestResolveDbService(unittest.TestCase): - def _resolve(self, app_id: str, deployed: tuple[str, ...]) -> str: + def _resolve( + self, app_id: str, deployed: tuple[str, ...], call: str = "resolve_db_service" + ) -> str: with tempfile.TemporaryDirectory() as tmp: stub_bin = Path(tmp) / "bin" stub_bin.mkdir() @@ -51,7 +54,7 @@ class TestResolveDbService(unittest.TestCase): BASH_ENV="", ) proc = subprocess.run( - ["bash", "-c", f'source "{CONTEXT}"; resolve_db_service'], + ["bash", "-c", f'source "{CONTEXT}"; {call}'], cwd=PROJECT_ROOT, env=env, capture_output=True, @@ -87,6 +90,20 @@ class TestResolveDbService(unittest.TestCase): def test_a_role_without_a_database_dep_resolves_nothing(self) -> None: self.assertEqual(self._resolve(NO_DB_APP, (CENTRAL_SERVICE,)), "") + def test_an_early_match_in_a_long_listing_still_resolves(self) -> None: + deployed = (LOCAL_SERVICE, *FILLER, CENTRAL_SERVICE) + for _ in range(3): + self.assertEqual(self._resolve(PINNED_ENGINE_APP, deployed), LOCAL_SERVICE) + + def test_an_early_match_in_a_long_listing_counts_as_deployed(self) -> None: + service = self._resolve(PINNED_ENGINE_APP, (), 'printf "%s" "${SERVICE_NAME}"') + deployed = (service, *FILLER) + call = "if has_swarm_service; then echo deployed; fi" + for _ in range(3): + self.assertEqual( + self._resolve(PINNED_ENGINE_APP, deployed, call), "deployed" + ) + if __name__ == "__main__": unittest.main()