diff --git a/playbooks/update/README.md b/playbooks/update/README.md index 3372c54b81..fa51710d17 100644 --- a/playbooks/update/README.md +++ b/playbooks/update/README.md @@ -1,7 +1,9 @@ -# Update Report 📋 +# Update 🔄 -This playbook collects the state of an Infinito.Nexus host of any version as the starting point of an update, and judges that state and the host's inventory against the code of this checkout. +These playbooks prepare the update of an Infinito.Nexus host of any version to the code of this checkout. +[report.yml](report.yml) collects the state of the host as the starting point of the update, and judges that state and the host's inventory against the code. It only reads and changes nothing on the host. +[pull.yml](pull.yml) pulls the images the deploy of the host runs, see [Pull](#pull-). ## Run 🚀 @@ -25,7 +27,7 @@ The host needs no Python, because both collectors run as plain shell scripts. ## Check 🚦 -[state.sh](files/state.sh) collects the host's state as tab-separated records, and the filter [update_check](../../plugins/filter/update_check.py) compares it with what `lookup('applications')` and `lookup('application_closure', group_names)` derive from the current code and the inventory. +[state.sh](files/state.sh) collects the host's state as tab-separated records, and the filter [update_check](../../plugins/filter/update/check.py) compares it with what `lookup('applications')` and `lookup('application_closure', group_names)` derive from the current code and the inventory. | Area | ❌ | ⚠️ | |---|---|---| @@ -60,3 +62,23 @@ Run it before an update to see the gap to the current code, and after the deploy | backups | Newest backup of every backup repository below the current and the `/Backups` layout | The report contains no container environments and no compose file contents. + +## Pull 📥 + +[pull.yml](pull.yml) pulls every image the deploy of the host runs onto the host, also while the host still runs an older version. +Run it from the repository root, with the `PYTHONPATH` prefix of [Run](#run-) in a worktree: + +```bash +ansible-playbook -i --vault-password-file -l playbooks/update/pull.yml +``` + +The filter [update_pull_images](../../plugins/filter/update/pull.py) lists every service with an `image` and a `version` that `lookup('applications')` gives a role of `lookup('application_closure', group_names)`, without the shared services another role provides, as the images area of the [Check](#check-) does. +A service that declares `custom` runs the `_custom` image its role builds: the playbook skips that image, lists it as built by the role, and pulls the `image` and `version` the service declares, the base of the build. +A role that builds an image without declaring `custom` names no built image in its configuration: its declared images are pulled, its built image is neither pulled nor listed. +The list holds no sidecar image a compose template takes from a role outside the closure: the redis of a role with an embedded redis or an SSO proxy (`svc-db-redis`), the memcached of OpenProject and Zammad (`svc-db-memcached`), an app-local database its role pins no image for (`svc-db-postgres`, `svc-db-mariadb`), and a local object store. +The deploy pulls these small images itself when the host lacks them. + +[pull.sh](files/pull.sh) runs `docker pull` for every image, also for one the host already has, and touches no container. +A failed pull is retried three times, ten seconds apart, and an attempt that runs longer than an hour, scaled by `TIMEOUT_FACTOR`, fails without a retry. +The play ends with the images pulled, the images whose pull changed nothing, the images the roles build, and every image whose pull still failed with the registry's error. +One failed image fails the play, so `failed=0` means every listed image is present on the host. diff --git a/playbooks/update/files/pull.sh b/playbooks/update/files/pull.sh new file mode 100755 index 0000000000..db55160cf3 --- /dev/null +++ b/playbooks/update/files/pull.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Args: +# $1 image reference to pull +# Output: "pulled" when the pull changed the local image, "present" when the host already had it. +set -euo pipefail + +REF="${1:?image reference required}" + +before="$(docker image inspect --format '{{.Id}}' "$REF" 2>/dev/null)" || before="" +docker pull --quiet "$REF" >/dev/null +after="$(docker image inspect --format '{{.Id}}' "$REF")" + +if [[ "$before" == "$after" ]]; then + echo present +else + echo pulled +fi diff --git a/playbooks/update/pull.yml b/playbooks/update/pull.yml new file mode 100644 index 0000000000..1a2515daf4 --- /dev/null +++ b/playbooks/update/pull.yml @@ -0,0 +1,26 @@ +--- +- name: "📥 Pull every image the deploy of the host runs" + hosts: all + become: true + gather_facts: false + tasks: + - name: "🧮 Derive the images the deploy of the host runs" + ansible.builtin.set_fact: + update_pull_plan: "{{ lookup('applications') | update_pull_images(lookup('application_closure', group_names)) }}" + + - name: "📥 Pull the images" + block: + - name: "📥 Pull the image" + ansible.builtin.script: + cmd: "pull.sh {{ item | quote }}" + loop: "{{ update_pull_plan.pull }}" + register: update_pull_runs + changed_when: update_pull_runs.stdout | trim == 'pulled' + retries: 3 + delay: 10 + until: update_pull_runs is succeeded + timeout: "{{ lookup('timeout', 3600) | int }}" + always: + - name: "📊 Show what the pull did" + ansible.builtin.debug: + msg: "{{ update_pull_runs.results | update_pull_summary(update_pull_plan.built) }}" diff --git a/plugins/filter/update/__init__.py b/plugins/filter/update/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/filter/update_check.py b/plugins/filter/update/check.py similarity index 100% rename from plugins/filter/update_check.py rename to plugins/filter/update/check.py diff --git a/plugins/filter/update/pull.py b/plugins/filter/update/pull.py new file mode 100644 index 0000000000..5a35f85eb1 --- /dev/null +++ b/plugins/filter/update/pull.py @@ -0,0 +1,103 @@ +"""Filters ``update_pull_images`` and ``update_pull_summary``: the images a deploy of a host runs, and what pulling them did. + +update_pull_images: + Args: + applications: ``lookup('applications')`` of the host. + deployed: ``lookup('application_closure', group_names)``, the roles a + deploy of the host runs. + + Returns: + ``{"pull": [...], "built": [...]}``: ``pull`` holds the sorted pull + reference of every image a deployed role pins for a service it runs + itself, ``built`` every image such a service builds locally, as + `` (, ...)``. + +update_pull_summary: + Args: + results: ``results`` of the loop that ran ``pull.sh`` over ``pull``. + built: ``built`` of ``update_pull_images``. + + Returns: + The counts, the references per outcome, and one line per failed + reference with its error. +""" + +from __future__ import annotations + +from collections import defaultdict +from functools import partial + +from utils.docker.image.custom import custom_image_name +from utils.docker.image.pin import pull_reference +from utils.roles.entity.name import get_entity_name +from utils.update_check.common import FAIL, OK, pinned, services + +PULLED = "pulled" +PRESENT = "present" + + +def update_pull_images(applications, deployed): + apps = sorted(app for app in deployed if app in applications) + pull = { + pull_reference(str(service["image"]).strip(), service["version"]) + for app in apps + for _, service in pinned(app, applications[app]) + } + built = defaultdict(set) + for app in apps: + for key, service in services(app, applications[app]): + if not service.get("custom"): + continue + name = custom_image_name(service["custom"], partial(get_entity_name, app)) + if not name: + raise ValueError(f"{app}: services.{key}.custom names no image") + built[name].add(app) + return { + "pull": sorted(pull), + "built": [ + f"{name} ({', '.join(sorted(roles))})" + for name, roles in sorted(built.items()) + ], + } + + +def _error(result) -> str: + text = result.get("stderr") or result["msg"] + return " ".join(line.strip() for line in str(text).splitlines() if line.strip()) + + +def update_pull_summary(results, built): + done = defaultdict(list) + failed = [] + for result in results: + reference = str(result["item"]) + if result.get("failed"): + failed.append(f"{FAIL} {reference}: {_error(result)}") + continue + outcome = str(result["stdout"]).strip() + if outcome not in (PULLED, PRESENT): + raise ValueError(f"{reference}: pull.sh printed {outcome!r}") + done[outcome].append(reference) + counts = " ".join( + [ + f"⬇️ {len(done[PULLED])} pulled", + f"{OK} {len(done[PRESENT])} already present", + f"⏭️ {len(built)} built by role", + f"{FAIL} {len(failed)} failed", + ] + ) + return [ + counts, + f"⬇️ pulled: {', '.join(done[PULLED])}", + f"{OK} already present: {', '.join(done[PRESENT])}", + f"⏭️ built by role: {', '.join(built)}", + *failed, + ] + + +class FilterModule: + def filters(self): + return { + "update_pull_images": update_pull_images, + "update_pull_summary": update_pull_summary, + } diff --git a/plugins/lookup/image.py b/plugins/lookup/image.py index 02cd1cef95..10a4f4c2cf 100644 --- a/plugins/lookup/image.py +++ b/plugins/lookup/image.py @@ -51,12 +51,14 @@ Both terms (application_id, service_key) are required. from __future__ import annotations import contextlib +from functools import partial from typing import Any from ansible.errors import AnsibleError from ansible.plugins.loader import lookup_loader from ansible.plugins.lookup import LookupBase +from utils.docker.image.custom import custom_image_name from utils.roles.entity.name import get_entity_name try: @@ -164,16 +166,14 @@ class LookupModule(LookupBase): image_value = _as_str(image_override) if not image_value and custom: - if isinstance(custom, str) and custom.strip(): - custom_base = custom.strip() - else: - custom_base = _as_str(get_entity_name(application_id)) - if not custom_base: + image_value = custom_image_name( + custom, partial(get_entity_name, application_id) + ) + if not image_value: raise AnsibleError( f"image: cannot derive entity name from " f"application_id '{application_id}' for custom=True" ) - image_value = f"{custom_base}_custom" if not image_value: image_value = _as_str(entry.get("image")) if not image_value: diff --git a/tests/unit/python/plugins/filter/update/__init__.py b/tests/unit/python/plugins/filter/update/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/plugins/filter/test_update_check.py b/tests/unit/python/plugins/filter/update/test_check.py similarity index 99% rename from tests/unit/python/plugins/filter/test_update_check.py rename to tests/unit/python/plugins/filter/update/test_check.py index adf33d91b4..0b4830c1fa 100644 --- a/tests/unit/python/plugins/filter/test_update_check.py +++ b/tests/unit/python/plugins/filter/update/test_check.py @@ -1,6 +1,6 @@ import unittest -from plugins.filter.update_check import FilterModule, update_check +from plugins.filter.update.check import FilterModule, update_check from utils.update_check.common import FAIL, OK, WARN NOW = 1_800_000_000 diff --git a/tests/unit/python/plugins/filter/update/test_pull.py b/tests/unit/python/plugins/filter/update/test_pull.py new file mode 100644 index 0000000000..4fff364426 --- /dev/null +++ b/tests/unit/python/plugins/filter/update/test_pull.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from plugins.filter.update.pull import ( + FilterModule, + update_pull_images, + update_pull_summary, +) +from utils.update_check.common import FAIL, OK + +DEPLOYED = [ + "svc-db-postgres", + "svc-db-redis", + "sys-ctl-x", + "web-app-nextcloud", + "web-app-wordpress", +] + + +def _applications(): + return { + "svc-db-postgres": { + "services": { + "postgres": { + "image": "postgres", + "version": "18", + "shared": True, + "custom": True, + } + } + }, + "svc-db-redis": {"services": {"redis": {"image": "redis", "version": "8"}}}, + "web-app-nextcloud": { + "services": { + "nextcloud": { + "image": "nextcloud", + "version": "34-fpm-alpine", + "custom": True, + }, + "cron": {"custom": True}, + "whiteboard": { + "image": "ghcr.io/nextcloud-releases/whiteboard", + "version": "v1.5.9", + "custom": "nextcloud-whiteboard", + }, + "proxy": {"image": "nginx", "version": "1.31.8"}, + "notify": {"image": "nginx"}, + "redis": { + "enabled": True, + "shared": True, + "image": "redis", + "version": "7", + }, + "talk": { + "enabled": "False", + "image": "nextcloud/aio-talk", + "version": "latest", + }, + } + }, + "web-app-wordpress": { + "services": { + "wordpress": {"image": "wordpress", "version": "6.8"}, + "proxy": {"image": "nginx", "version": "1.31.8"}, + } + }, + "web-app-unused": {"services": {"unused": {"image": "unused", "version": "1"}}}, + } + + +def _images(): + return update_pull_images(_applications(), DEPLOYED) + + +def _result(item, stdout="", **fields): + return {"item": item, "failed": False, "stdout": stdout, **fields} + + +class TestUpdatePullImages(unittest.TestCase): + def test_filters_are_registered(self): + filters = FilterModule().filters() + self.assertIs(filters["update_pull_images"], update_pull_images) + self.assertIs(filters["update_pull_summary"], update_pull_summary) + + def test_every_pinned_image_of_a_deployed_role_is_pulled(self): + self.assertEqual( + _images()["pull"], + [ + "ghcr.io/nextcloud-releases/whiteboard:v1.5.9", + "nextcloud:34-fpm-alpine", + "nginx:1.31.8", + "postgres:18", + "redis:8", + "wordpress:6.8", + ], + ) + + def test_the_base_of_a_locally_built_image_is_pulled(self): + pull = _images()["pull"] + self.assertIn("nextcloud:34-fpm-alpine", pull) + self.assertIn("postgres:18", pull) + + def test_a_shared_service_another_role_provides_is_not_pulled(self): + self.assertNotIn("redis:7", _images()["pull"]) + + def test_a_disabled_service_is_not_pulled(self): + self.assertNotIn("nextcloud/aio-talk:latest", _images()["pull"]) + + def test_an_image_without_version_is_not_pulled(self): + self.assertNotIn("nginx", _images()["pull"]) + + def test_a_role_outside_the_closure_is_not_pulled(self): + self.assertNotIn("unused:1", _images()["pull"]) + + def test_an_image_two_roles_pin_is_pulled_once(self): + self.assertEqual(_images()["pull"].count("nginx:1.31.8"), 1) + + def test_a_digest_pin_is_pulled_by_digest(self): + applications = { + "svc-db-redis": { + "services": {"redis": {"image": "redis", "version": "sha256:abc"}} + } + } + self.assertEqual( + update_pull_images(applications, ["svc-db-redis"])["pull"], + ["redis@sha256:abc"], + ) + + def test_built_lists_every_locally_built_image_with_its_roles(self): + self.assertEqual( + _images()["built"], + [ + "nextcloud-whiteboard_custom (web-app-nextcloud)", + "nextcloud_custom (web-app-nextcloud)", + "postgres_custom (svc-db-postgres)", + ], + ) + + def test_a_custom_service_without_a_name_fails(self): + with ( + patch("plugins.filter.update.pull.get_entity_name", return_value=""), + self.assertRaises(ValueError), + ): + _images() + + +class TestUpdatePullSummary(unittest.TestCase): + def setUp(self): + self.summary = update_pull_summary( + [ + _result("nextcloud:34-fpm-alpine", "pulled\n"), + _result("nginx:1.31.8", "present\n"), + _result("postgres:18", "present\n"), + _result( + "redis:8", + failed=True, + msg="non-zero return code", + stderr="Error response from daemon: manifest unknown\n", + ), + ], + ["nextcloud_custom (web-app-nextcloud)"], + ) + + def test_the_first_line_counts_every_outcome(self): + self.assertEqual( + self.summary[0], + f"⬇️ 1 pulled {OK} 2 already present " + f"⏭️ 1 built by role {FAIL} 1 failed", + ) + + def test_every_outcome_lists_its_images(self): + self.assertIn("⬇️ pulled: nextcloud:34-fpm-alpine", self.summary) + self.assertIn(f"{OK} already present: nginx:1.31.8, postgres:18", self.summary) + self.assertIn( + "⏭️ built by role: nextcloud_custom (web-app-nextcloud)", self.summary + ) + + def test_a_failed_pull_names_the_image_and_the_registry_error(self): + self.assertIn( + f"{FAIL} redis:8: Error response from daemon: manifest unknown", + self.summary, + ) + + def test_a_failure_without_stderr_names_its_message(self): + summary = update_pull_summary( + [_result("redis:8", failed=True, msg="The task timed out")], [] + ) + self.assertIn(f"{FAIL} redis:8: The task timed out", summary) + + def test_unexpected_output_fails(self): + with self.assertRaises(ValueError): + update_pull_summary([_result("redis:8", "downloaded\n")], []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/docker/test_image_custom.py b/tests/unit/python/utils/docker/test_image_custom.py new file mode 100644 index 0000000000..5706b34f05 --- /dev/null +++ b/tests/unit/python/utils/docker/test_image_custom.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import unittest +from unittest.mock import Mock + +from utils.docker.image.custom import custom_image_name + + +class TestCustomImageName(unittest.TestCase): + def test_true_names_the_image_after_the_entity(self): + self.assertEqual( + custom_image_name(True, lambda: "mattermost"), "mattermost_custom" + ) + + def test_a_string_names_the_image_itself(self): + self.assertEqual( + custom_image_name("nextcloud-whiteboard", lambda: "nextcloud"), + "nextcloud-whiteboard_custom", + ) + + def test_a_string_never_asks_for_the_entity(self): + entity_name = Mock(return_value="nextcloud") + custom_image_name("side", entity_name) + entity_name.assert_not_called() + + def test_surrounding_whitespace_is_dropped(self): + self.assertEqual(custom_image_name(" side ", lambda: "x"), "side_custom") + self.assertEqual( + custom_image_name(True, lambda: " mattermost "), "mattermost_custom" + ) + + def test_a_blank_string_names_the_image_after_the_entity(self): + self.assertEqual( + custom_image_name(" ", lambda: "mattermost"), "mattermost_custom" + ) + + def test_no_base_names_no_image(self): + self.assertEqual(custom_image_name(True, lambda: ""), "") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/update_check/__init__.py b/tests/unit/python/utils/update_check/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/utils/update_check/test_common.py b/tests/unit/python/utils/update_check/test_common.py new file mode 100644 index 0000000000..544261a395 --- /dev/null +++ b/tests/unit/python/utils/update_check/test_common.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import unittest + +from utils.update_check.common import pinned + + +def _keys(application_id, services): + return [key for key, _ in pinned(application_id, {"services": services})] + + +class TestPinned(unittest.TestCase): + def test_a_service_with_image_and_version_is_pinned(self): + services = {"nextcloud": {"image": "nextcloud", "version": "34-fpm-alpine"}} + self.assertEqual(_keys("web-app-nextcloud", services), ["nextcloud"]) + + def test_a_service_without_version_is_not_pinned(self): + services = {"proxy": {"image": "nginx"}} + self.assertEqual(_keys("web-app-nextcloud", services), []) + + def test_a_service_without_image_is_not_pinned(self): + services = {"cron": {"version": "34-fpm-alpine", "custom": True}} + self.assertEqual(_keys("web-app-nextcloud", services), []) + + def test_a_disabled_service_is_not_pinned(self): + services = {"talk": {"enabled": "False", "image": "talk", "version": "1"}} + self.assertEqual(_keys("web-app-nextcloud", services), []) + + def test_a_shared_service_of_another_role_is_not_pinned(self): + services = {"redis": {"shared": True, "image": "redis", "version": "8"}} + self.assertEqual(_keys("web-app-nextcloud", services), []) + + def test_the_shared_primary_service_is_pinned(self): + services = {"postgres": {"shared": True, "image": "postgres", "version": "18"}} + self.assertEqual(_keys("svc-db-postgres", services), ["postgres"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/docker/image/custom.py b/utils/docker/image/custom.py new file mode 100644 index 0000000000..a830921a69 --- /dev/null +++ b/utils/docker/image/custom.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + + +def custom_image_name(custom, entity_name: Callable[[], str]) -> str: + """Name of the image a service declaring ``custom`` builds locally. + + Args: + custom: the service's truthy ``custom`` value. A non-empty string names + the image, any other value names it after the role's entity. + entity_name: returns the entity name of the role that builds the image, + called only when ``custom`` names no image itself. + + Returns: + ``_custom``, or ``""`` when the base is empty. + """ + if isinstance(custom, str) and custom.strip(): + base = custom.strip() + else: + base = str(entity_name()).strip() + return f"{base}_custom" if base else "" diff --git a/utils/update_check/common.py b/utils/update_check/common.py index 711fa942cd..b6a162f5ea 100644 --- a/utils/update_check/common.py +++ b/utils/update_check/common.py @@ -83,3 +83,9 @@ def services(application_id: str, application: dict): if key != entity and truthy(service.get("shared")): continue yield key, service + + +def pinned(application_id: str, application: dict): + for key, service in services(application_id, application): + if service.get("image") and service.get("version"): + yield key, service diff --git a/utils/update_check/storage.py b/utils/update_check/storage.py index ca165ca0b8..d19dd9bb23 100644 --- a/utils/update_check/storage.py +++ b/utils/update_check/storage.py @@ -11,6 +11,7 @@ from utils.update_check.common import ( WARN, compare, owner, + pinned, repo, services, version, @@ -171,10 +172,9 @@ def data(report, applications, records, expected): def images(report, applications, deployed, records): wanted: dict[tuple[str, str], set[str]] = defaultdict(set) for app in deployed: - for _, service in services(app, applications[app]): - if service.get("image") and service.get("version"): - key = (app, repo(str(service["image"]))[0]) - wanted[key].add(str(service["version"])) + for _, service in pinned(app, applications[app]): + key = (app, repo(str(service["image"]))[0]) + wanted[key].add(str(service["version"])) for fields in records["container"]: if len(fields) < 3: continue