diff --git a/meta/packages.yml b/meta/packages.yml index 1ae37d7c40..1059056f49 100644 --- a/meta/packages.yml +++ b/meta/packages.yml @@ -120,6 +120,7 @@ iproute: - iproute2 RedHat: - iproute + batch: false iputils-ping: Archlinux: @@ -128,6 +129,7 @@ iputils-ping: - iputils-ping RedHat: - iputils + batch: false java-jdk: Archlinux: @@ -179,6 +181,7 @@ nfs-client: - nfs-common RedHat: - nfs-utils + batch: false php: Archlinux: @@ -198,6 +201,7 @@ postfix: RedHat: - postfix - postfix-lmdb + batch: false retext: Archlinux: diff --git a/plugins/action/package_install.py b/plugins/action/package_install.py index ea7fe211c1..aa9b103b99 100755 --- a/plugins/action/package_install.py +++ b/plugins/action/package_install.py @@ -17,7 +17,8 @@ role states what it needs and never how to get it:: Distribution repositories, the AUR, COPR, PPA and source builds all run through here, including the unprivileged build environment the AUR needs. :mod:`utils.packages.plan` decides which module calls that takes; this -plugin only executes them. +plugin only executes them. Ids whose plans only call the host's package +manager share one call, so a long list costs one transaction. Args: id: logical package id, or a list of ids, each declared either in the @@ -43,7 +44,7 @@ from utils.packages.calls import ( ) from utils.packages.facts import distribution_and_family, read_fact from utils.packages.outcome import aggregate -from utils.packages.plan import build_plan +from utils.packages.plan import build_plan, merge_plans from utils.packages.registry import build_registry, project_root_from_env, resolve from utils.packages.schema import ROLE_FILE_META_PACKAGES, PackagesShapeError @@ -68,7 +69,7 @@ class ActionModule(ActionBase): registry = self._registry() role = self._role() - results: list[dict[str, Any]] = [] + plans: list[list[ModuleCall]] = [] skipped: list[str] = [] for package_id in package_ids: @@ -77,6 +78,10 @@ class ActionModule(ActionBase): if not plan: skipped.append(package_id) continue + plans.append(plan) + + results: list[dict[str, Any]] = [] + for plan in merge_plans(plans): results.extend(self._run_plan(plan, task_vars, facts)) return aggregate(results, skipped, distribution) diff --git a/plugins/lookup/package_batch.py b/plugins/lookup/package_batch.py new file mode 100644 index 0000000000..108bdc76a2 --- /dev/null +++ b/plugins/lookup/package_batch.py @@ -0,0 +1,58 @@ +"""Lookup ``package_batch``: the package ids a host installs in one run at +deploy start. + + {{ lookup('package_batch', group_names) }} + +The terms are every group of the host, after the swarm auto-placement, so a +role the service loader reaches on this host through any whitelist is covered. They are widened to +every role the deploy may run on the host +(:func:`utils.roles.invocations.deploy_closure`), and the sorted ids the +batch installs for those roles (:func:`utils.packages.batch.batch_ids`) are +returned. +""" + +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleError +from ansible.plugins.loader import lookup_loader +from ansible.plugins.lookup import LookupBase + +from utils import PROJECT_ROOT +from utils.packages.batch import batch_ids +from utils.packages.schema import PackagesShapeError +from utils.roles.invocations import deploy_closure +from utils.roles.validation.invokable import types_from_group_names + + +class LookupModule(LookupBase): + def run( + self, + terms: list[Any] | None, + variables: dict[str, Any] | None = None, + **kwargs: Any, + ) -> list[list[str]]: + seeds: list[str] = [] + for term in terms or []: + if isinstance(term, (list, tuple, set)): + seeds.extend(str(entry) for entry in term) + elif term: + seeds.append(str(term)) + + vars_ = variables or getattr(self._templar, "available_variables", {}) or {} + group_names = [str(group) for group in vars_["group_names"]] + applications = lookup_loader.get( + "applications", loader=self._loader, templar=getattr(self, "_templar", None) + ).run([], variables=vars_)[0] + try: + roles = deploy_closure( + applications, + seeds, + project_root=PROJECT_ROOT, + host_types=types_from_group_names(group_names), + group_names=group_names, + ) + return [batch_ids(roles, PROJECT_ROOT)] + except (PackagesShapeError, TypeError, ValueError) as exc: + raise AnsibleError(f"package_batch: {exc}") from exc diff --git a/roles/desk-bluray-player/tasks/main.yml b/roles/desk-bluray-player/tasks/main.yml index 6ecbb33cec..ed97d539c0 100644 --- a/roles/desk-bluray-player/tasks/main.yml +++ b/roles/desk-bluray-player/tasks/main.yml @@ -1,4 +1 @@ --- -- name: 📦 install vlc and blu ray software - package_install: - id: vlc-bluray diff --git a/roles/desk-chromium/tasks/main.yml b/roles/desk-chromium/tasks/main.yml index 8effd2fe30..af710359ab 100644 --- a/roles/desk-chromium/tasks/main.yml +++ b/roles/desk-chromium/tasks/main.yml @@ -1,8 +1,4 @@ --- -- name: 📦 Install Chromium browser - package_install: - id: chromium - - name: 📂 Include dependency 'desk-keepassxc' include_role: name: desk-keepassxc diff --git a/roles/desk-copyq/tasks/main.yml b/roles/desk-copyq/tasks/main.yml index 3423d40ffe..cdb589cb65 100644 --- a/roles/desk-copyq/tasks/main.yml +++ b/roles/desk-copyq/tasks/main.yml @@ -1,8 +1,4 @@ --- -- name: 📦 Install CopyQ clipboard manager - package_install: - id: copyq - - name: ✅ Ensure autostart directory exists file: path: "{{ ansible_facts['env']['HOME'] }}/.config/autostart" diff --git a/roles/desk-firefox/tasks/main.yml b/roles/desk-firefox/tasks/main.yml index 7d80f2529d..fb7d61fc7e 100644 --- a/roles/desk-firefox/tasks/main.yml +++ b/roles/desk-firefox/tasks/main.yml @@ -1,8 +1,4 @@ --- -- name: 📦 Install Firefox - package_install: - id: firefox - - name: ✅ Ensure Firefox distribution policies directory exists file: path: "{{ firefox_policy_dir }}" diff --git a/roles/desk-git/meta/packages.yml b/roles/desk-git/meta/packages.yml new file mode 100644 index 0000000000..2844b63173 --- /dev/null +++ b/roles/desk-git/meta/packages.yml @@ -0,0 +1,3 @@ +--- +git: + shared: true diff --git a/roles/desk-git/tasks/main.yml b/roles/desk-git/tasks/main.yml index 011b2ff645..ee99abefe4 100644 --- a/roles/desk-git/tasks/main.yml +++ b/roles/desk-git/tasks/main.yml @@ -1,8 +1,4 @@ --- -- name: "📦 Install git" - package_install: - id: git - - name: 📦 Install gitcon CLI system-wide via pip include_role: name: sys-pip-install diff --git a/roles/desk-gnome-caffeine/tasks/00_core.yml b/roles/desk-gnome-caffeine/tasks/00_core.yml index b012f7fdff..ea7af3ae51 100644 --- a/roles/desk-gnome-caffeine/tasks/00_core.yml +++ b/roles/desk-gnome-caffeine/tasks/00_core.yml @@ -1,8 +1,4 @@ --- -- name: 📦 Install caffeine - package_install: - id: caffeine - - name: 📁 Create autostart directory if it doesn't exist file: path: "{{auto_start_directory}}" diff --git a/roles/desk-gnome-terminal/tasks/00_core.yml b/roles/desk-gnome-terminal/tasks/00_core.yml index 9cd48ebb07..398994cfa3 100644 --- a/roles/desk-gnome-terminal/tasks/00_core.yml +++ b/roles/desk-gnome-terminal/tasks/00_core.yml @@ -1,8 +1,4 @@ --- -- name: 📦 Install Gnome Terminal - package_install: - id: gnome-terminal - - name: "🔧 Set zsh as default shell for {{ lookup('users', WORKSTATION_USER).username }}" user: name: "{{ lookup('users', WORKSTATION_USER).username }}" diff --git a/roles/desk-gnome/tasks/00_core.yml b/roles/desk-gnome/tasks/00_core.yml index 3df89b9be9..09eac0f064 100644 --- a/roles/desk-gnome/tasks/00_core.yml +++ b/roles/desk-gnome/tasks/00_core.yml @@ -1,9 +1,4 @@ --- -- name: 📦 Install GNOME GSettings schemas + GNOME Shell (cross-distro) - become: true - package_install: - id: gnome-shell - - name: 📂 Include dependencies include_role: name: "{{ desk_gnome_dependency }}" diff --git a/roles/desk-gnucash/tasks/main.yml b/roles/desk-gnucash/tasks/main.yml index 2823e94977..ed97d539c0 100644 --- a/roles/desk-gnucash/tasks/main.yml +++ b/roles/desk-gnucash/tasks/main.yml @@ -1,4 +1 @@ --- -- name: 📦 Install GnuCash - package_install: - id: gnucash diff --git a/roles/desk-jrnl/tasks/main.yml b/roles/desk-jrnl/tasks/main.yml index b3f745285a..ed97d539c0 100644 --- a/roles/desk-jrnl/tasks/main.yml +++ b/roles/desk-jrnl/tasks/main.yml @@ -1,4 +1 @@ --- -- name: 📦 install jrnl - package_install: - id: jrnl diff --git a/roles/desk-keepassxc/tasks/main.yml b/roles/desk-keepassxc/tasks/main.yml index 31cc7abdfc..ce1131213f 100644 --- a/roles/desk-keepassxc/tasks/main.yml +++ b/roles/desk-keepassxc/tasks/main.yml @@ -1,8 +1,4 @@ --- - block: - - name: "📦 Install keepassxc" - package_install: - id: keepassxc - - include_tasks: utils/once/flag.yml when: run_once_desk_keepassxc is not defined diff --git a/roles/desk-libreoffice/tasks/main.yml b/roles/desk-libreoffice/tasks/main.yml index 52cfdc9b61..7f1ceea7df 100644 --- a/roles/desk-libreoffice/tasks/main.yml +++ b/roles/desk-libreoffice/tasks/main.yml @@ -1,8 +1,4 @@ --- -- name: 📦 install libreoffice - package_install: - id: libreoffice - - name: 📦 install libreoffice language packages package: # nocheck: dynamic-package (the language pack name is built from the configured flavor and the loop variable) name: "libreoffice-{{ lookup('applications', 'desk-libreoffice').flavor }}-{{ item }}" diff --git a/roles/desk-micro/tasks/main.yml b/roles/desk-micro/tasks/main.yml index 5dbeb2ccf2..ed97d539c0 100644 --- a/roles/desk-micro/tasks/main.yml +++ b/roles/desk-micro/tasks/main.yml @@ -1,4 +1 @@ --- -- name: 📦 install micro - package_install: - id: micro diff --git a/roles/desk-neovim/tasks/main.yml b/roles/desk-neovim/tasks/main.yml index 19e6d6c1a3..ed97d539c0 100644 --- a/roles/desk-neovim/tasks/main.yml +++ b/roles/desk-neovim/tasks/main.yml @@ -1,4 +1 @@ --- -- name: 📦 install neovim - package_install: - id: neovim diff --git a/roles/desk-nextcloud/tasks/main.yml b/roles/desk-nextcloud/tasks/main.yml index a630b7975f..9b988d0ede 100644 --- a/roles/desk-nextcloud/tasks/main.yml +++ b/roles/desk-nextcloud/tasks/main.yml @@ -1,8 +1,4 @@ --- -- name: 📦 Install nextcloud-client - package_install: - id: nextcloud-desktop - - name: 🎬 Initialize dotlinker mapping list ansible.builtin.set_fact: nextcloud_dotlinker_mappings: [] diff --git a/roles/desk-obs/tasks/main.yml b/roles/desk-obs/tasks/main.yml index 53f464a646..ed97d539c0 100644 --- a/roles/desk-obs/tasks/main.yml +++ b/roles/desk-obs/tasks/main.yml @@ -1,4 +1 @@ --- -- name: 📦 install streaming - package_install: - id: obs-studio diff --git a/roles/desk-qbittorrent/README.md b/roles/desk-qbittorrent/README.md index 4f037b9810..f88a5b0011 100644 --- a/roles/desk-qbittorrent/README.md +++ b/roles/desk-qbittorrent/README.md @@ -69,7 +69,7 @@ infinito administration deploy dedicated "$INVENTORY/devices.yml" \ The `main.yml` file in the `desk-qbittorrent` role includes the following task: 1. **Install Torrent Software**: - - This task calls `package_install` with the `qbittorrent` id; the per-distribution names are declared in [`meta/packages.yml`](meta/packages.yml). + - The package batch the constructor runs at deploy start installs the `qbittorrent` id; the per-distribution names are declared in [`meta/packages.yml`](meta/packages.yml). ## Purpose and Usage diff --git a/roles/desk-qbittorrent/tasks/main.yml b/roles/desk-qbittorrent/tasks/main.yml index d56b09ceb5..739472e6f8 100644 --- a/roles/desk-qbittorrent/tasks/main.yml +++ b/roles/desk-qbittorrent/tasks/main.yml @@ -1,8 +1,4 @@ --- - block: - - name: 📦 install torrent software - package_install: - id: qbittorrent - - include_tasks: utils/once/flag.yml when: run_once_desk_qbittorrent is not defined diff --git a/roles/desk-retroarch/tasks/main.yml b/roles/desk-retroarch/tasks/main.yml index 161f60f2c8..ed97d539c0 100644 --- a/roles/desk-retroarch/tasks/main.yml +++ b/roles/desk-retroarch/tasks/main.yml @@ -1,4 +1 @@ --- -- name: 📦 Install RetroArch and assets - package_install: - id: retroarch diff --git a/roles/desk-spotify/README.md b/roles/desk-spotify/README.md index aa597baad4..90cafccb82 100644 --- a/roles/desk-spotify/README.md +++ b/roles/desk-spotify/README.md @@ -6,7 +6,7 @@ This Ansible role installs the [Spotify](https://www.spotify.com/) desktop clien ## Overview -Spotify is a digital music streaming service that gives you access to millions of songs and podcasts. The role calls `package_install` with the `spotify` id; [`meta/packages.yml`](meta/packages.yml) declares it as an AUR package on Arch Linux and as a deliberate no-op on the Debian and RedHat families, which do not archive the proprietary client. +Spotify is a digital music streaming service that gives you access to millions of songs and podcasts. The package batch the constructor runs at deploy start installs the `spotify` id; [`meta/packages.yml`](meta/packages.yml) declares it as an AUR package on Arch Linux and as a deliberate no-op on the Debian and RedHat families, which do not archive the proprietary client. ## Cosmos diff --git a/roles/desk-spotify/tasks/main.yml b/roles/desk-spotify/tasks/main.yml index 94410d3c08..9cf077e9f7 100644 --- a/roles/desk-spotify/tasks/main.yml +++ b/roles/desk-spotify/tasks/main.yml @@ -1,8 +1,4 @@ --- - block: - - name: 📦 install spotify - package_install: - id: spotify - - include_tasks: utils/once/flag.yml when: run_once_desk_spotify is not defined diff --git a/roles/desk-torbrowser/tasks/main.yml b/roles/desk-torbrowser/tasks/main.yml index 413f664817..ed97d539c0 100644 --- a/roles/desk-torbrowser/tasks/main.yml +++ b/roles/desk-torbrowser/tasks/main.yml @@ -1,4 +1 @@ --- -- name: 📦 Install torbrowser - package_install: - id: torbrowser diff --git a/roles/desk-zoom/README.md b/roles/desk-zoom/README.md index 61c8095dc4..12197944ee 100644 --- a/roles/desk-zoom/README.md +++ b/roles/desk-zoom/README.md @@ -6,7 +6,7 @@ ## Overview -This role installs Zoom through `package_install` with the `zoom` id. [`meta/packages.yml`](meta/packages.yml) declares it as an AUR package on Arch Linux and as a deliberate no-op on the Debian and RedHat families, which do not archive the proprietary client. +The package batch the constructor runs at deploy start installs Zoom through the `zoom` id. [`meta/packages.yml`](meta/packages.yml) declares it as an AUR package on Arch Linux and as a deliberate no-op on the Debian and RedHat families, which do not archive the proprietary client. ## Cosmos diff --git a/roles/desk-zoom/tasks/main.yml b/roles/desk-zoom/tasks/main.yml index bae4ad0ba4..87c815aa16 100644 --- a/roles/desk-zoom/tasks/main.yml +++ b/roles/desk-zoom/tasks/main.yml @@ -1,8 +1,4 @@ --- - block: - - name: 📦 install video conference software - package_install: - id: zoom - - include_tasks: utils/once/flag.yml when: run_once_desk_zoom is not defined diff --git a/roles/dev-arduino/tasks/main.yml b/roles/dev-arduino/tasks/main.yml index 014a08da9a..071019fc53 100644 --- a/roles/dev-arduino/tasks/main.yml +++ b/roles/dev-arduino/tasks/main.yml @@ -1,8 +1,4 @@ --- -- name: 📦 install arduino developer tools - package_install: - id: arduino-ide - - name: "🔧 Add user {{ lookup('users', WORKSTATION_USER).username }} to arduino groups" ansible.builtin.user: name: "{{ lookup('users', WORKSTATION_USER).username }}" diff --git a/roles/dev-gcc/meta/packages.yml b/roles/dev-gcc/meta/packages.yml index 48d2291f03..cb7af00134 100644 --- a/roles/dev-gcc/meta/packages.yml +++ b/roles/dev-gcc/meta/packages.yml @@ -9,3 +9,4 @@ gcc: RedHat: - gcc - libstdc++ + batch: false diff --git a/roles/dev-git/meta/packages.yml b/roles/dev-git/meta/packages.yml new file mode 100644 index 0000000000..2844b63173 --- /dev/null +++ b/roles/dev-git/meta/packages.yml @@ -0,0 +1,3 @@ +--- +git: + shared: true diff --git a/roles/dev-git/tasks/main.yml b/roles/dev-git/tasks/main.yml index 2785241996..0fd1085f6a 100644 --- a/roles/dev-git/tasks/main.yml +++ b/roles/dev-git/tasks/main.yml @@ -1,8 +1,4 @@ --- - block: - - name: "📦 Install git" - package_install: - id: git - - include_tasks: utils/once/flag.yml when: run_once_dev_git is not defined diff --git a/roles/dev-java/meta/packages.yml b/roles/dev-java/meta/packages.yml new file mode 100644 index 0000000000..53b5125418 --- /dev/null +++ b/roles/dev-java/meta/packages.yml @@ -0,0 +1,3 @@ +--- +java-jdk: + shared: true diff --git a/roles/dev-java/tasks/main.yml b/roles/dev-java/tasks/main.yml index 699f16e551..ed97d539c0 100644 --- a/roles/dev-java/tasks/main.yml +++ b/roles/dev-java/tasks/main.yml @@ -1,4 +1 @@ --- -- name: 📦 install java - package_install: - id: java-jdk diff --git a/roles/dev-make/tasks/main.yml b/roles/dev-make/tasks/main.yml index dbdec40ac3..ed97d539c0 100644 --- a/roles/dev-make/tasks/main.yml +++ b/roles/dev-make/tasks/main.yml @@ -1,4 +1 @@ --- -- name: 📦 install make - package_install: - id: make diff --git a/roles/dev-nix/meta/packages.yml b/roles/dev-nix/meta/packages.yml index da680255fb..be1557b404 100644 --- a/roles/dev-nix/meta/packages.yml +++ b/roles/dev-nix/meta/packages.yml @@ -1,6 +1,10 @@ --- +ca-certificates: + shared: true + nix: Archlinux: - nix Debian: [] # nocheck: empty - installed from the official releases.nixos.org installer on this family; TODO: switch to the nix-bin and nix-setup-systemd packages if the installer is ever dropped RedHat: [] # nocheck: empty - installed from the official releases.nixos.org installer on this family; TODO: switch to the EPEL nix package if the installer is ever dropped + batch: false diff --git a/roles/dev-nix/tasks/00_core.yml b/roles/dev-nix/tasks/00_core.yml index bcd81755ae..b8bb85235e 100644 --- a/roles/dev-nix/tasks/00_core.yml +++ b/roles/dev-nix/tasks/00_core.yml @@ -3,12 +3,6 @@ # - Arch-based systems: install via package manager # - Non-Arch systems: use the official installer with SHA256 verification -# 0) Ensure CA trust is present and discover CA bundle path for Nix TLS -- name: "📦 Install the CA trust store" - package_install: - id: ca-certificates - become: true - - name: 🔍 Probe CA bundle candidate paths ansible.builtin.stat: path: "{{ item }}" diff --git a/roles/dev-nodejs/tasks/main.yml b/roles/dev-nodejs/tasks/main.yml index bdd31754a1..ed97d539c0 100644 --- a/roles/dev-nodejs/tasks/main.yml +++ b/roles/dev-nodejs/tasks/main.yml @@ -1,4 +1 @@ --- -- name: ✅ Ensure Node.js is installed - package_install: - id: nodejs diff --git a/roles/dev-npm/tasks/main.yml b/roles/dev-npm/tasks/main.yml index f1e13976ba..1d05f75cca 100644 --- a/roles/dev-npm/tasks/main.yml +++ b/roles/dev-npm/tasks/main.yml @@ -1,8 +1,4 @@ --- -- name: ✅ Ensure npm is installed - package_install: - id: npm - - name: ▶️ Run 'npm ci' command: npm ci args: diff --git a/roles/drv-epson-multiprinter/tasks/00_core.yml b/roles/drv-epson-multiprinter/tasks/00_core.yml index 5f304ab01c..70042adc45 100644 --- a/roles/drv-epson-multiprinter/tasks/00_core.yml +++ b/roles/drv-epson-multiprinter/tasks/00_core.yml @@ -1,6 +1,2 @@ --- -- name: 📦 install epson printer and scanner drivers - package_install: - id: epson-multiprinter - - include_tasks: utils/once/flag.yml diff --git a/roles/drv-intel/tasks/main.yml b/roles/drv-intel/tasks/main.yml index ef0be3870e..ed97d539c0 100644 --- a/roles/drv-intel/tasks/main.yml +++ b/roles/drv-intel/tasks/main.yml @@ -1,4 +1 @@ --- -- name: 📦 Install Intel media driver package(s) - package_install: - id: intel-media-driver diff --git a/roles/drv-non-free/tasks/main.yml b/roles/drv-non-free/tasks/main.yml index 1b5e6b3313..f34f13471f 100644 --- a/roles/drv-non-free/tasks/main.yml +++ b/roles/drv-non-free/tasks/main.yml @@ -3,11 +3,6 @@ ansible.builtin.setup: when: ansible_facts is not defined -- name: ✅ Ensure mhwd is installed on Manjaro - package_install: - id: mhwd - become: true - - name: 🔍 Detect mhwd command ansible.builtin.stat: path: /usr/bin/mhwd diff --git a/roles/gen-hunspell/tasks/main.yml b/roles/gen-hunspell/tasks/main.yml index 1024aff1ad..daf9331192 100644 --- a/roles/gen-hunspell/tasks/main.yml +++ b/roles/gen-hunspell/tasks/main.yml @@ -1,8 +1,4 @@ --- -- name: 📦 install hunspell - package_install: - id: hunspell - - name: 📦 install hunspell language packages package: # nocheck: dynamic-package name: "hunspell-{{ item }}" diff --git a/roles/svc-bkp-volume-2-local/meta/packages.yml b/roles/svc-bkp-volume-2-local/meta/packages.yml index 420cd30850..3cc18c9b93 100644 --- a/roles/svc-bkp-volume-2-local/meta/packages.yml +++ b/roles/svc-bkp-volume-2-local/meta/packages.yml @@ -6,6 +6,7 @@ btrfs-tools: - btrfs-progs RedHat: - btrfs-progs + batch: false centos: names: - btrfs-progs diff --git a/roles/svc-db-openldap/tasks/_python_ldap_prerequisites.yml b/roles/svc-db-openldap/tasks/_python_ldap_prerequisites.yml index a0be0a467a..150185b83c 100644 --- a/roles/svc-db-openldap/tasks/_python_ldap_prerequisites.yml +++ b/roles/svc-db-openldap/tasks/_python_ldap_prerequisites.yml @@ -1,8 +1,4 @@ --- -- name: 📦 Install python-ldap native library build dependencies - package_install: - id: openldap-dev - - name: 🔍 Resolve Python include directory for active interpreter ansible.builtin.command: argv: diff --git a/roles/svc-net-firewall/tasks/01_host.yml b/roles/svc-net-firewall/tasks/01_host.yml index 9474747829..b8a08ae231 100644 --- a/roles/svc-net-firewall/tasks/01_host.yml +++ b/roles/svc-net-firewall/tasks/01_host.yml @@ -1,9 +1,4 @@ --- -- name: "📦 Firewall | Install nftables" - become: true - package_install: - id: nftables - - name: "📁 Firewall | Ensure the fragment directory exists" become: true ansible.builtin.file: diff --git a/roles/svc-net-tor/meta/packages.yml b/roles/svc-net-tor/meta/packages.yml index b5bdbd8924..349a9a6e91 100644 --- a/roles/svc-net-tor/meta/packages.yml +++ b/roles/svc-net-tor/meta/packages.yml @@ -6,3 +6,4 @@ dnsmasq: - dnsmasq RedHat: - dnsmasq + batch: false diff --git a/roles/svc-net-wireguard/meta/packages.yml b/roles/svc-net-wireguard/meta/packages.yml index ca3794afe1..996f701343 100644 --- a/roles/svc-net-wireguard/meta/packages.yml +++ b/roles/svc-net-wireguard/meta/packages.yml @@ -9,3 +9,4 @@ wireguard: RedHat: - procps-ng - wireguard-tools + batch: false diff --git a/roles/svc-opt-keyboard-color/tasks/00_core.yml b/roles/svc-opt-keyboard-color/tasks/00_core.yml index 789029b4ea..f5ea5919cf 100644 --- a/roles/svc-opt-keyboard-color/tasks/00_core.yml +++ b/roles/svc-opt-keyboard-color/tasks/00_core.yml @@ -2,10 +2,6 @@ - include_role: name: sys-ctl-alm-compose -- name: 📦 Install MSI packages - package_install: - id: msi-perkeyrgb - - include_role: name: sys-service vars: diff --git a/roles/svc-storage-nfs-server/meta/packages.yml b/roles/svc-storage-nfs-server/meta/packages.yml index d0af599556..dde066d0ce 100644 --- a/roles/svc-storage-nfs-server/meta/packages.yml +++ b/roles/svc-storage-nfs-server/meta/packages.yml @@ -6,11 +6,13 @@ e2fsprogs: - e2fsprogs RedHat: - e2fsprogs + batch: false libntirpc: Archlinux: [] # nocheck: empty - the AUR build compiles its own ntirpc submodule; TODO: revisit if the AUR build ever links against a system ntirpc Debian: [] # nocheck: empty - the packaged nfs-ganesha links against a matching libntirpc, only Fedora ships a broken pair; TODO: revisit if Debian ever ships a mismatched nfs-ganesha and libntirpc pair RedHat: [] # nocheck: empty - only the Fedora override needs the upgrade; the CentOS SIG build is self-consistent; TODO: revisit if the CentOS SIG build ever needs the same upgrade as Fedora + batch: false fedora: names: - libntirpc @@ -29,6 +31,7 @@ nfs-ganesha: RedHat: - nfs-ganesha - nfs-ganesha-vfs + batch: false centos: names: - nfs-ganesha @@ -41,9 +44,11 @@ nfs-kernel-server: Debian: - nfs-kernel-server RedHat: [] # nocheck: empty - the kernel server ships inside nfs-utils, installed through the nfs-client id; TODO: revisit if the family ever splits the kernel server out of nfs-utils + batch: false selinux-python-binding: Archlinux: [] # nocheck: empty - SELinux is not the mandatory access control on this family; TODO: implement the AppArmor equivalent if this family ever needs export confinement Debian: [] # nocheck: empty - SELinux is not the mandatory access control on this family; TODO: implement the AppArmor equivalent if this family ever needs export confinement RedHat: - python3-libselinux + batch: false diff --git a/roles/sys-ctl-alm-telegram/meta/packages.yml b/roles/sys-ctl-alm-telegram/meta/packages.yml index 04726d4501..f09f817ef7 100644 --- a/roles/sys-ctl-alm-telegram/meta/packages.yml +++ b/roles/sys-ctl-alm-telegram/meta/packages.yml @@ -6,3 +6,4 @@ curl: - curl RedHat: - curl + batch: false diff --git a/roles/sys-ctl-cln-bkps/meta/packages.yml b/roles/sys-ctl-cln-bkps/meta/packages.yml index 1a477501b1..f269a69d0e 100644 --- a/roles/sys-ctl-cln-bkps/meta/packages.yml +++ b/roles/sys-ctl-cln-bkps/meta/packages.yml @@ -6,3 +6,4 @@ lsof: - lsof RedHat: - lsof + batch: false diff --git a/roles/sys-ctl-mtn-cert-renew/meta/packages.yml b/roles/sys-ctl-mtn-cert-renew/meta/packages.yml index f1ab745a72..8831fb925b 100644 --- a/roles/sys-ctl-mtn-cert-renew/meta/packages.yml +++ b/roles/sys-ctl-mtn-cert-renew/meta/packages.yml @@ -6,6 +6,7 @@ certbot-nginx: - python3-certbot-nginx RedHat: - python3-certbot-nginx + batch: false centos: names: - python3-certbot-nginx diff --git a/roles/sys-daemon/tasks/00_core.yml b/roles/sys-daemon/tasks/00_core.yml index 88e6d4b0c9..924b698a5d 100644 --- a/roles/sys-daemon/tasks/00_core.yml +++ b/roles/sys-daemon/tasks/00_core.yml @@ -1,9 +1,4 @@ --- -- name: 📦 Install systemd tools (provides systemd-analyze) - package_install: - id: systemd-tools - become: true - - name: "♻️ reset (if enabled) for {{ role_name}}" include_tasks: 01_reset.yml when: MODE_RESET | bool and run_once_sys_daemon is not defined diff --git a/roles/sys-hostname/meta/packages.yml b/roles/sys-hostname/meta/packages.yml index 4ab1f2083e..0a09d6a8b3 100644 --- a/roles/sys-hostname/meta/packages.yml +++ b/roles/sys-hostname/meta/packages.yml @@ -6,3 +6,4 @@ hostname-tool: - hostname RedHat: - hostname + batch: false diff --git a/roles/sys-openssl/tasks/00_core.yml b/roles/sys-openssl/tasks/00_core.yml index a326b83c1c..70042adc45 100644 --- a/roles/sys-openssl/tasks/00_core.yml +++ b/roles/sys-openssl/tasks/00_core.yml @@ -1,7 +1,2 @@ --- -- name: 📦 Install OpenSSL - become: true - package_install: - id: openssl - - include_tasks: utils/once/flag.yml diff --git a/roles/sys-pip/meta/packages.yml b/roles/sys-pip/meta/packages.yml index 9ebe8bb79f..1d2fa9b3f9 100644 --- a/roles/sys-pip/meta/packages.yml +++ b/roles/sys-pip/meta/packages.yml @@ -6,3 +6,4 @@ python-pip: - python3-pip RedHat: - python3-pip + batch: false diff --git a/roles/sys-sudo/meta/packages.yml b/roles/sys-sudo/meta/packages.yml index a717eb47e5..60c3110a7b 100644 --- a/roles/sys-sudo/meta/packages.yml +++ b/roles/sys-sudo/meta/packages.yml @@ -6,3 +6,4 @@ sudo: - sudo RedHat: - sudo + batch: false diff --git a/roles/sys-svc-certbot/meta/packages.yml b/roles/sys-svc-certbot/meta/packages.yml index 97831723f1..709f9f52a3 100644 --- a/roles/sys-svc-certbot/meta/packages.yml +++ b/roles/sys-svc-certbot/meta/packages.yml @@ -6,6 +6,7 @@ certbot: - certbot RedHat: - certbot + batch: false centos: names: - certbot diff --git a/roles/sys-svc-compose/tasks/00_core.yml b/roles/sys-svc-compose/tasks/00_core.yml index d1e8dd2992..14d22062e5 100644 --- a/roles/sys-svc-compose/tasks/00_core.yml +++ b/roles/sys-svc-compose/tasks/00_core.yml @@ -23,10 +23,6 @@ owner: root group: root -- name: "✅ Ensure compose wrapper system deps (PyYAML)" - package_install: - id: python-yaml - - name: 📦 Install compose wrapper become: true ansible.builtin.copy: diff --git a/roles/sys-svc-container/meta/packages.yml b/roles/sys-svc-container/meta/packages.yml index 56315d6b55..3bceb3a5b2 100644 --- a/roles/sys-svc-container/meta/packages.yml +++ b/roles/sys-svc-container/meta/packages.yml @@ -23,3 +23,4 @@ docker-cli: - iptables-nft repo: managed_externally: sys-svc-container configures the download.docker.com repository per distro before installing + batch: false diff --git a/roles/sys-svc-mail-msmtp/meta/packages.yml b/roles/sys-svc-mail-msmtp/meta/packages.yml index 83552f61fb..31d552efeb 100644 --- a/roles/sys-svc-mail-msmtp/meta/packages.yml +++ b/roles/sys-svc-mail-msmtp/meta/packages.yml @@ -1,4 +1,7 @@ --- +ca-certificates: + shared: true + msmtp: Archlinux: - msmtp @@ -18,3 +21,4 @@ msmtp-mta: Debian: - msmtp-mta RedHat: [] # nocheck: empty - neither the EL base repositories nor EPEL ship msmtp-mta; the role links /usr/bin/sendmail to msmtp instead; TODO: revisit if EPEL ever ships msmtp-mta, which would replace the sendmail symlink + batch: false diff --git a/roles/sys-svc-mail-msmtp/tasks/enable.yml b/roles/sys-svc-mail-msmtp/tasks/enable.yml index add9db8054..0167145a21 100644 --- a/roles/sys-svc-mail-msmtp/tasks/enable.yml +++ b/roles/sys-svc-mail-msmtp/tasks/enable.yml @@ -13,14 +13,6 @@ is the dedicated NFS server (svc-storage-nfs-server). Hosts outside the swarm (svc-swarm-node) keep a local {{ DIR_VAR_LIB }}. -- name: 📦 Install msmtp base package - package_install: - id: msmtp - -- name: "📦 Install the CA trust store" - package_install: - id: ca-certificates - - name: ✅ Ensure the shared Root CA is trusted for the TLS mail relay include_role: name: sys-ca-selfsigned diff --git a/roles/sys-svc-nss/tasks/00_core.yml b/roles/sys-svc-nss/tasks/00_core.yml index 1a883ff42f..f32335ca2d 100644 --- a/roles/sys-svc-nss/tasks/00_core.yml +++ b/roles/sys-svc-nss/tasks/00_core.yml @@ -1,9 +1,4 @@ --- -- name: ✅ Ensure nss-myhostname is installed so *.localhost resolves to loopback - become: true - package_install: - id: nss-myhostname - - name: ✅ Ensure nsswitch resolves hosts via myhostname become: true ansible.builtin.lineinfile: diff --git a/roles/sys-svc-ssh/meta/packages.yml b/roles/sys-svc-ssh/meta/packages.yml index 1784d762f9..672cd80498 100644 --- a/roles/sys-svc-ssh/meta/packages.yml +++ b/roles/sys-svc-ssh/meta/packages.yml @@ -6,3 +6,4 @@ openssh-client: - openssh-client RedHat: - openssh-clients + batch: false diff --git a/roles/user-backup/meta/packages.yml b/roles/user-backup/meta/packages.yml index 41f7ad032a..f240de6bf7 100644 --- a/roles/user-backup/meta/packages.yml +++ b/roles/user-backup/meta/packages.yml @@ -6,3 +6,4 @@ acl: - acl RedHat: - acl + batch: false diff --git a/roles/web-app-discourse/tasks/02_docker.yml b/roles/web-app-discourse/tasks/02_docker.yml index acb7801e9a..c6cbe7a537 100644 --- a/roles/web-app-discourse/tasks/02_docker.yml +++ b/roles/web-app-discourse/tasks/02_docker.yml @@ -1,8 +1,4 @@ --- -- name: 📦 install which - package_install: - id: which - - name: "🧩 load docker, db and proxy for {{ application_id }}" include_role: name: sys-stk-full diff --git a/tasks/stages/01_constructor.yml b/tasks/stages/01_constructor.yml index 9055c529c4..b94ba02122 100644 --- a/tasks/stages/01_constructor.yml +++ b/tasks/stages/01_constructor.yml @@ -177,6 +177,20 @@ applications_carrier: {} when: DEPLOYMENT_MODE == 'swarm' +- name: "📦 Install the packages of every role this host may run, in one run" + package_install: + id: "{{ _package_batch_ids }}" + vars: + _package_batch_inventory: "{{ PACKAGES | default([]) }}" + _package_batch_ids: >- + {{ + ( + lookup('package_batch', group_names) + + ([_package_batch_inventory] if _package_batch_inventory is string else _package_batch_inventory) + ) | unique + }} + when: _package_batch_ids | length > 0 + - name: "🥾 Bootstrap svc-swarm-node" ansible.builtin.include_role: name: svc-swarm-node @@ -231,8 +245,3 @@ loop_control: loop_var: role_group label: "{{ role_group }}" - -- name: 📦 Install inventory-defined packages - package_install: - id: "{{ PACKAGES }}" - when: PACKAGES | default([]) | length > 0 diff --git a/tests/integration/ansible/lookups/test_package_batch.py b/tests/integration/ansible/lookups/test_package_batch.py new file mode 100644 index 0000000000..b870584156 --- /dev/null +++ b/tests/integration/ansible/lookups/test_package_batch.py @@ -0,0 +1,96 @@ +"""Integration test: package_batch against the real merged config and role tree. + +The host deploys the application set of a local Nextcloud deploy. The batch +must hold the ids of the supporting roles that deploy ran through +``include_role`` (none of them is an application of the host), and nothing a +role still installs itself or only a workstation runs. +""" + +from __future__ import annotations + +import unittest + +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar + +from plugins.lookup.applications import _reset_cache_for_tests +from plugins.lookup.package_batch import LookupModule as PackageBatchLookup +from utils.packages.registry import build_registry + +from . import PROJECT_ROOT + +APPLICATIONS = [ + "svc-bkp-secrets-2-local", + "svc-bkp-volume-2-local", + "svc-db-mariadb", + "svc-db-openldap", + "svc-db-postgres", + "svc-db-redis", + "web-app-keycloak", + "web-app-nextcloud", + "web-app-seaweedfs", + "web-svc-cdn", + "web-svc-collabora", + "web-svc-coturn", + "web-svc-logout", +] +SUPPORTING_ROLE_IDS = { + "sys-daemon": "systemd-tools", + "sys-openssl": "openssl", + "sys-svc-compose": "python-yaml", + "sys-svc-mail-msmtp": "msmtp", + "sys-svc-nss": "nss-myhostname", + "svc-db-openldap": "openldap-dev", +} +WORKSTATION_ID = "firefox" + + +def _variables(group_names: list[str]) -> dict: + return { + "applications": {}, + "users": {}, + "DOMAIN_PRIMARY": "infinito.test", + "SYSTEM_EMAIL_DOMAIN": "infinito.test", + "DIR_COMPOSITIONS": "/opt/compose/", + "group_names": list(group_names), + } + + +class TestPackageBatchIntegration(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + _reset_cache_for_tests() + lookup = PackageBatchLookup() + lookup._loader = DataLoader() + lookup._templar = Templar(loader=lookup._loader) + cls.batch = lookup.run([APPLICATIONS], variables=_variables(APPLICATIONS))[0] + cls.registry = build_registry(PROJECT_ROOT) + + @classmethod + def tearDownClass(cls) -> None: + _reset_cache_for_tests() + + def test_supporting_roles_are_covered(self) -> None: + missing = { + role: package_id + for role, package_id in SUPPORTING_ROLE_IDS.items() + if package_id not in self.batch + } + self.assertEqual( + missing, + {}, + "The deploy runs these roles through include_role only; the batch " + "dropped their ids, so nothing installs them any more.", + ) + + def test_the_batch_holds_no_unbatched_id(self) -> None: + self.assertEqual( + [pid for pid in self.batch if not self.registry[pid].batch], [] + ) + + def test_a_server_gets_no_workstation_package(self) -> None: + self.assertNotIn(WORKSTATION_ID, self.batch) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/lint/ansible/modules/README.md b/tests/lint/ansible/modules/README.md index 7a042abd42..bc0e12d35a 100644 --- a/tests/lint/ansible/modules/README.md +++ b/tests/lint/ansible/modules/README.md @@ -10,6 +10,7 @@ Tests in this directory MUST only cover Ansible module / task usage rules. - `test_no_legacy_uri_module_usage.py` forbids the unqualified `uri:` module name; tasks MUST use `ansible.builtin.uri`. - `test_no_literal_no_log.py` forbids literal `no_log: true` / `no_log: false`; every gate MUST flow through the `MASK_CREDENTIALS_IN_LOGS` variable. - `test_no_redundant_default_on_module_io.py` forbids `| default(...)` on a task's own `register:` output or input arguments inside its own conditionals, where the value is provably set. +- `test_package_batch.py` requires that a role's own `package_install` tasks install only ids flagged `batch: false` in `meta/packages.yml`, that every id of a role the constructor runs before the deploy-start batch carries that flag, and that a `shared: true` reference names a root id. - `test_run_once_tags.py` requires that every `set_fact` marked `run_once: true` carries the canonical run-once tags so re-runs and `--tags` filters keep the fact populated. Shell-pipeline rules and SQL formatting rules live in sibling directories; see `../shell/` and `../sql/`. diff --git a/tests/lint/ansible/modules/test_package_batch.py b/tests/lint/ansible/modules/test_package_batch.py new file mode 100644 index 0000000000..116d57c1b1 --- /dev/null +++ b/tests/lint/ansible/modules/test_package_batch.py @@ -0,0 +1,197 @@ +"""Lint: a role installs only the package ids the deploy-start batch leaves out. + +The constructor installs every id of every role a deploy may run in one +``package_install`` run (``lookup('package_batch')``). An id stays out of it +only when its ``meta/packages.yml`` entry says ``batch: false``, and then the +role's own task installs it. Four rules hold that split together: + +* an unconditional ``package_install`` task installs only ``batch: false`` + ids; any other id is covered by the batch, so the task goes; +* a ``package_install`` task under a ``when:``, on the task or an enclosing + block, installs only ``batch: false`` ids, so the batch does not install + them on hosts where the condition is false; +* every id of a role the constructor runs ahead of the batch is + ``batch: false``, because the batch would install it too late; +* a role file references only ids the shared root file declares. + +A ``run_once_`` guard is no condition here: it only keeps a role from +running twice in one deploy. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path +from typing import Any + +from utils.cache.files import iter_project_files +from utils.cache.yaml import load_yaml_any +from utils.packages.registry import build_registry, load_references +from utils.roles.invocations import ( + CONSTRUCTOR_STAGE, + NESTED_KEYS, + STAGES_DIR, + expand_roles, + iter_tasks, + role_names, + task_invocations, +) + +from . import PROJECT_ROOT + +_RUN_ONCE_GUARD_RE = re.compile( + r"^(?:run_once_\w+\s+is\s+not\s+defined" + r"|not\s+\(\s*run_once_\w+\s*\|\s*default\(false\)\s*\|\s*bool\s*\))$" +) +_BATCH_LOOKUP = "lookup('package_batch'" + + +def _conditions(task: dict[str, Any]) -> list[str]: + when = task.get("when") + if when is None: + return [] + return [ + str(condition).strip() + for condition in (when if isinstance(when, list) else [when]) + if not _RUN_ONCE_GUARD_RE.match(str(condition).strip()) + ] + + +def _package_tasks( + node: Any, inherited: list[str] +) -> list[tuple[dict[str, Any], list[str]]]: + found: list[tuple[dict[str, Any], list[str]]] = [] + if not isinstance(node, list): + return found + for task in node: + if not isinstance(task, dict): + continue + conditions = inherited + _conditions(task) + if isinstance(task.get("package_install"), dict): + found.append((task, conditions)) + for key in NESTED_KEYS: + found.extend(_package_tasks(task.get(key), conditions)) + return found + + +def _ids(task: dict[str, Any]) -> list[str]: + raw = task["package_install"].get("id") + ids = raw if isinstance(raw, list) else [raw] + return [str(item).strip() for item in ids if isinstance(item, str)] + + +def _role_task_files() -> list[Path]: + roles = PROJECT_ROOT / "roles" + return [ + Path(path) + for path in iter_project_files(extensions=(".yml",)) + if Path(path).is_relative_to(roles) + and Path(path).relative_to(roles).parts[1:2] in (("tasks",), ("handlers",)) + ] + + +def _constructor_tasks_ahead_of_the_batch() -> list[dict[str, Any]] | None: + tasks = load_yaml_any(str(PROJECT_ROOT / STAGES_DIR / CONSTRUCTOR_STAGE)) + for index, task in enumerate(tasks): + if "package_install" in task and _BATCH_LOOKUP in str(task.get("vars")): + return tasks[:index] + return None + + +def _roles_ahead_of_the_batch(tasks: list[dict[str, Any]]) -> set[str]: + known = role_names(str(PROJECT_ROOT / "roles")) + seeds: set[str] = set() + for task in iter_tasks(tasks): + seeds |= task_invocations(task, known)[0] + return expand_roles({}, seeds, project_root=PROJECT_ROOT) + + +class TestPackageBatch(unittest.TestCase): + def setUp(self) -> None: + self.registry = build_registry(PROJECT_ROOT) + + def _offenders(self, *, conditional: bool) -> list[str]: + offenders: list[str] = [] + for path in _role_task_files(): + doc = load_yaml_any(str(path), default_if_missing=None) + for task, conditions in _package_tasks(doc, []): + if bool(conditions) != conditional: + continue + batched = [ + package_id + for package_id in _ids(task) + if package_id in self.registry and self.registry[package_id].batch + ] + if batched: + rel = path.relative_to(PROJECT_ROOT) + offenders.append( + f"{rel}: '{task.get('name', 'unnamed')}' installs " + f"{', '.join(batched)}" + ) + return offenders + + def test_unconditional_tasks_install_only_unbatched_ids(self) -> None: + offenders = self._offenders(conditional=False) + if offenders: + self.fail( + f"{len(offenders)} unconditional package_install task(s) install an " + "id the deploy-start batch already installs. Remove the task, or " + "flag the id 'batch: false' in its meta/packages.yml when it must " + "wait for a setup step the role runs first:\n" + + "\n".join(f" - {o}" for o in offenders) + ) + + def test_conditional_tasks_install_only_unbatched_ids(self) -> None: + offenders = self._offenders(conditional=True) + if offenders: + self.fail( + f"{len(offenders)} package_install task(s) under a when: install an " + "id the deploy-start batch installs unconditionally. Flag the id " + "'batch: false' in its meta/packages.yml:\n" + + "\n".join(f" - {o}" for o in offenders) + ) + + def test_roles_ahead_of_the_batch_install_only_unbatched_ids(self) -> None: + tasks = _constructor_tasks_ahead_of_the_batch() + self.assertIsNotNone( + tasks, + f"{STAGES_DIR / CONSTRUCTOR_STAGE} has no package_install task fed by " + f"{_BATCH_LOOKUP}...), so no deploy installs the batched ids.", + ) + references = load_references(PROJECT_ROOT) + offenders: list[str] = [] + for role in sorted(_roles_ahead_of_the_batch(tasks)): + owned = { + package_id + for package_id, declaration in self.registry.items() + if declaration.role == role + } + for package_id in sorted(owned | references.get(role, set())): + declaration = self.registry.get(package_id) + if declaration is not None and declaration.batch: + offenders.append(f"{role}: {package_id}") + if offenders: + self.fail( + f"{len(offenders)} id(s) belong to a role the constructor runs before " + "the deploy-start batch, so the batch installs them too late. Flag " + "each 'batch: false' in its meta/packages.yml:\n" + + "\n".join(f" - {o}" for o in offenders) + ) + + def test_references_name_shared_ids(self) -> None: + offenders = [ + f"roles/{role}/meta/packages.yml: '{package_id}'" + for role, package_ids in sorted(load_references(PROJECT_ROOT).items()) + for package_id in sorted(package_ids) + if package_id not in self.registry or not self.registry[package_id].shared + ] + if offenders: + self.fail( + f"{len(offenders)} 'shared: true' reference(s) name no id of the " + "root meta/packages.yml:\n" + "\n".join(f" - {o}" for o in offenders) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/plugins/action/test_package_install.py b/tests/unit/python/plugins/action/test_package_install.py index 9cc8b29085..b45cbdbb4a 100644 --- a/tests/unit/python/plugins/action/test_package_install.py +++ b/tests/unit/python/plugins/action/test_package_install.py @@ -2,10 +2,12 @@ import unittest import unittest.mock as mock from ansible.errors import AnsibleActionFail +from ansible.plugins.action import ActionBase from ansible.plugins.loader import become_loader, shell_loader from plugins.action.package_install import ActionModule from utils.packages.calls import GENERIC_PACKAGE, ModuleCall, RetryPolicy +from utils.packages.schema import PackageSpec def _action(args) -> ActionModule: @@ -238,6 +240,47 @@ class TestTargetFacts(unittest.TestCase): self.assertIn("localhost", str(caught.exception)) +class TestRunMergesPlainPlans(unittest.TestCase): + FACTS = {"distribution": "Debian", "os_family": "Debian", "pkg_mgr": "apt"} + + def _run(self, package_ids, specs): + action = _action({"id": package_ids}) + action._task._role = None + action._task.delegate_to = None + action._connection = mock.Mock(become=None, transport="local") + action._execute_module = mock.Mock(return_value={"changed": False}) + registry = { + package_id: mock.Mock(package_id=package_id, role=None, shared=True) + for package_id in package_ids + } + with ( + mock.patch.object(ActionBase, "run", return_value={}), + mock.patch( + "plugins.action.package_install.build_registry", return_value=registry + ), + mock.patch( + "plugins.action.package_install.resolve", + side_effect=lambda declaration, *_: specs[declaration.package_id], + ), + ): + result = action.run(task_vars={"ansible_facts": self.FACTS}) + return action._execute_module, result + + def test_two_repository_ids_cost_one_package_manager_call(self): + execute, _result = self._run( + ["git", "curl"], + {"git": PackageSpec(("git",)), "curl": PackageSpec(("curl",))}, + ) + execute.assert_called_once() + self.assertEqual(execute.call_args.kwargs["module_args"]["name"], ["git", "curl"]) + + def test_an_id_without_names_is_reported_skipped(self): + _execute, result = self._run( + ["git", "mhwd"], {"git": PackageSpec(("git",)), "mhwd": PackageSpec(())} + ) + self.assertEqual(result["skipped_ids"], ["mhwd"]) + + class TestAttempt(unittest.TestCase): def _action_with(self, results): action = _action({}) diff --git a/tests/unit/python/plugins/lookup/test_package_batch.py b/tests/unit/python/plugins/lookup/test_package_batch.py new file mode 100644 index 0000000000..207854df7d --- /dev/null +++ b/tests/unit/python/plugins/lookup/test_package_batch.py @@ -0,0 +1,52 @@ +import unittest +from unittest import mock + +from ansible.errors import AnsibleError + +from plugins.lookup.package_batch import LookupModule +from utils.packages.schema import PackagesShapeError + +MODULE = "plugins.lookup.package_batch" + + +class TestPackageBatchLookup(unittest.TestCase): + def _run(self, terms, group_names, closure=None, side_effect=None): + applications = mock.Mock() + applications.run.return_value = [{"web-app-x": {}}] + with ( + mock.patch(f"{MODULE}.lookup_loader.get", return_value=applications), + mock.patch(f"{MODULE}.types_from_group_names", return_value=["server"]), + mock.patch( + f"{MODULE}.deploy_closure", + return_value=closure or {"web-app-x"}, + side_effect=side_effect, + ) as closure_mock, + mock.patch(f"{MODULE}.batch_ids", return_value=["git"]) as ids_mock, + ): + lookup = LookupModule() + result = lookup.run(terms, variables={"group_names": group_names}) + return result, closure_mock, ids_mock + + def test_returns_the_batch_ids_of_the_closure(self): + result, _closure, ids = self._run([["web-app-x"]], ["web-app-x"]) + self.assertEqual(result, [["git"]]) + self.assertEqual(ids.call_args.args[0], {"web-app-x"}) + + def test_nested_and_scalar_terms_become_one_seed_list(self): + _result, closure, _ids = self._run([["web-app-x"], "role-b"], ["web-app-x"]) + self.assertEqual(closure.call_args.args[1], ["web-app-x", "role-b"]) + + def test_the_host_groups_select_types_and_gates(self): + _result, closure, _ids = self._run([["web-app-x"]], ["web-app-x", "servers"]) + self.assertEqual(closure.call_args.kwargs["host_types"], ["server"]) + self.assertEqual( + closure.call_args.kwargs["group_names"], ["web-app-x", "servers"] + ) + + def test_a_registry_error_becomes_an_ansible_error(self): + with self.assertRaises(AnsibleError): + self._run([["web-app-x"]], [], side_effect=PackagesShapeError("broken")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/packages/test_batch.py b/tests/unit/python/utils/packages/test_batch.py new file mode 100644 index 0000000000..430b9eee2a --- /dev/null +++ b/tests/unit/python/utils/packages/test_batch.py @@ -0,0 +1,76 @@ +import tempfile +import unittest +from pathlib import Path + +from utils.cache.yaml import dump_yaml +from utils.packages.batch import batch_ids, batch_ids_by_role + + +def _write(root: Path, relative: str, payload) -> None: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + dump_yaml(str(path), payload) + + +class TestBatchIds(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + _write( + self.root, + "meta/packages.yml", + { + "git": {"Debian": ["git"]}, + "postfix": {"Debian": ["postfix"], "batch": False}, + }, + ) + + def test_a_role_batches_its_own_ids(self): + _write(self.root, "roles/role-a/meta/packages.yml", {"jq": {"Debian": ["jq"]}}) + self.assertEqual(batch_ids(["role-a"], self.root), ["jq"]) + + def test_an_unbatched_own_id_is_left_to_the_role(self): + _write( + self.root, + "roles/role-a/meta/packages.yml", + {"jq": {"Debian": ["jq"], "batch": False}}, + ) + self.assertEqual(batch_ids(["role-a"], self.root), []) + + def test_a_referenced_shared_id_is_batched_with_the_role(self): + _write(self.root, "roles/role-a/meta/packages.yml", {"git": {"shared": True}}) + self.assertEqual(batch_ids(["role-a"], self.root), ["git"]) + + def test_a_referenced_unbatched_shared_id_stays_out(self): + _write( + self.root, "roles/role-a/meta/packages.yml", {"postfix": {"shared": True}} + ) + self.assertEqual(batch_ids(["role-a"], self.root), []) + + def test_shared_ids_nobody_references_stay_out(self): + self.assertEqual(batch_ids(["role-a"], self.root), []) + + def test_roles_outside_the_deploy_contribute_nothing(self): + _write(self.root, "roles/role-a/meta/packages.yml", {"jq": {"Debian": ["jq"]}}) + _write( + self.root, "roles/role-b/meta/packages.yml", {"curl": {"Debian": ["curl"]}} + ) + self.assertEqual(batch_ids(["role-a"], self.root), ["jq"]) + + def test_ids_are_sorted_and_unique(self): + _write( + self.root, + "roles/role-a/meta/packages.yml", + {"jq": {"Debian": ["jq"]}, "git": {"shared": True}}, + ) + _write(self.root, "roles/role-b/meta/packages.yml", {"git": {"shared": True}}) + self.assertEqual(batch_ids(["role-b", "role-a"], self.root), ["git", "jq"]) + + def test_ids_are_grouped_by_role(self): + _write(self.root, "roles/role-a/meta/packages.yml", {"jq": {"Debian": ["jq"]}}) + self.assertEqual(batch_ids_by_role(self.root), {"role-a": {"jq"}}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/packages/test_plan.py b/tests/unit/python/utils/packages/test_plan.py index 30280bc3dd..990b002b6e 100644 --- a/tests/unit/python/utils/packages/test_plan.py +++ b/tests/unit/python/utils/packages/test_plan.py @@ -2,7 +2,7 @@ import re import unittest from utils.packages.calls import STATE_ABSENT, STATE_PRESENT -from utils.packages.plan import build_plan +from utils.packages.plan import build_plan, merge_plans from utils.packages.schema import SOURCE_AUR, SOURCE_BUILD, SOURCE_COPR, PackageSpec from utils.packages.sources import AUR_BUILDER_USER, PACMAN_CONF @@ -189,5 +189,53 @@ class TestBuildPlan(unittest.TestCase): ) +def _plan(*names, **spec): + return build_plan(PackageSpec(tuple(names), **spec), STATE_PRESENT) + + +def _names(call): + return call.args["name"] + + +class TestMergePlans(unittest.TestCase): + def test_plain_plans_share_one_call(self): + merged = merge_plans([_plan("git"), _plan("curl", "jq")]) + self.assertEqual(len(merged), 1) + self.assertEqual([_names(call) for call in merged[0]], [["git", "curl", "jq"]]) + + def test_a_name_requested_twice_is_installed_once(self): + merged = merge_plans([_plan("git"), _plan("git")]) + self.assertEqual(_names(merged[0][0]), ["git"]) + + def test_bootstrap_packages_install_before_every_name(self): + epel = {"repo": {"bootstrap_package": "epel-release"}} + merged = merge_plans([_plan("git"), _plan("micro", **epel), _plan("jq", **epel)]) + self.assertEqual( + [_names(call) for call in merged[0]], + [["epel-release"], ["git", "micro", "jq"]], + ) + + def test_a_plan_with_other_modules_runs_unchanged_after_the_merged_one(self): + aur = _plan("caffeine", source=SOURCE_AUR) + merged = merge_plans([aur, _plan("git")]) + self.assertEqual([_names(call) for call in merged[0]], [["git"]]) + self.assertEqual(merged[1], aur) + + def test_plans_of_different_states_never_merge(self): + absent = build_plan(PackageSpec(("postfix",)), STATE_ABSENT) + merged = merge_plans([_plan("git"), absent]) + self.assertEqual( + [(call.args["state"], _names(call)) for plan in merged for call in plan], + [(STATE_PRESENT, ["git"]), (STATE_ABSENT, ["postfix"])], + ) + + def test_no_plans_merge_to_nothing(self): + self.assertEqual(merge_plans([]), []) + + def test_the_merged_call_keeps_the_retry_policy(self): + merged = merge_plans([_plan("git"), _plan("curl")]) + self.assertIsNotNone(merged[0][0].retry) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/python/utils/packages/test_registry.py b/tests/unit/python/utils/packages/test_registry.py index cc631a1300..0892322ff3 100644 --- a/tests/unit/python/utils/packages/test_registry.py +++ b/tests/unit/python/utils/packages/test_registry.py @@ -7,6 +7,7 @@ from utils.packages.registry import ( build_registry, iter_inventory_package_lists, load_declarations, + load_references, resolve, ) from utils.packages.schema import ( @@ -223,6 +224,44 @@ class TestRegistry(unittest.TestCase): with self.assertRaises(PackagesShapeError): list(iter_inventory_package_lists(self.root)) + def test_an_unflagged_id_is_batched(self): + _write_role(self.root, "role-a", {"git": {"Debian": ["git"]}}) + self.assertTrue(self._declaration("git").batch) + + def test_batch_false_keeps_the_id_out_of_the_batch(self): + _write_role( + self.root, "role-a", {"git": {"Debian": ["git"], "batch": False}} + ) + self.assertFalse(self._declaration("git").batch) + + def test_the_batch_flag_is_no_distro_key(self): + _write_role( + self.root, "role-a", {"git": {"Debian": ["git"], "batch": False}} + ) + self.assertIsNone(resolve(self._declaration("git"), "arch")) + + def test_non_boolean_batch_raises(self): + _write_role(self.root, "role-a", {"git": {"Debian": ["git"], "batch": "no"}}) + with self.assertRaises(PackagesShapeError): + load_declarations(self.root) + + def test_a_shared_reference_is_no_declaration(self): + _write_shared(self.root, {"git": {"Debian": ["git"]}}) + _write_role(self.root, "role-a", {"git": {"shared": True}}) + declaration = self._declaration("git") + self.assertTrue(declaration.shared) + + def test_shared_references_are_collected_per_role(self): + _write_shared(self.root, {"git": {"Debian": ["git"]}}) + _write_role(self.root, "role-a", {"git": {"shared": True}}) + _write_role(self.root, "role-b", {"curl": {"Debian": ["curl"]}}) + self.assertEqual(load_references(self.root), {"role-a": {"git"}}) + + def test_shared_flag_in_the_root_file_is_no_reference(self): + _write_shared(self.root, {"git": {"shared": True}}) + self.assertEqual(load_references(self.root), {}) + self.assertIn("git", build_registry(self.root)) + def test_default_matrix_covers_five_distros(self): self.assertEqual( sorted(DISTRO_FAMILY), ["arch", "centos", "debian", "fedora", "ubuntu"] diff --git a/tests/unit/python/utils/roles/test_invocations.py b/tests/unit/python/utils/roles/test_invocations.py new file mode 100644 index 0000000000..489c8a9f4d --- /dev/null +++ b/tests/unit/python/utils/roles/test_invocations.py @@ -0,0 +1,245 @@ +import tempfile +import unittest +from pathlib import Path + +from utils.cache.yaml import dump_yaml +from utils.roles.invocations import ( + deploy_closure, + expand_roles, + role_invocations, + stage_files, + stage_invocations, +) +from utils.roles.mapping import ( + ROLE_FILE_HANDLERS_MAIN, + ROLE_FILE_META_MAIN, + ROLE_FILE_TASKS_MAIN, + ROLE_FILE_VARS_MAIN, +) + +ROLES = ("role-a", "role-b", "role-c", "sys-ctl-hlth-x", "sys-ctl-hlth-y") + + +def _role_file(role: str, role_file: str) -> str: + return f"roles/{role}/{role_file}" + + +class _TreeMixin: + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + for role in ROLES: + (self.root / "roles" / role / "tasks").mkdir(parents=True) + + def _write(self, relative: str, payload) -> Path: + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + dump_yaml(str(path), payload) + return path + + def _invoked(self, tasks, role: str = "role-a") -> frozenset[str]: + self._write(_role_file(role, ROLE_FILE_TASKS_MAIN), tasks) + return role_invocations(str(self.root / "roles"), role) + + +class TestRoleInvocations(_TreeMixin, unittest.TestCase): + def test_a_literal_include_role_is_an_edge(self): + self.assertEqual( + self._invoked([{"include_role": {"name": "role-b"}}]), {"role-b"} + ) + + def test_an_import_role_is_an_edge(self): + self.assertEqual( + self._invoked([{"ansible.builtin.import_role": {"name": "role-b"}}]), + {"role-b"}, + ) + + def test_a_literal_loop_names_every_item(self): + tasks = [ + { + "include_role": {"name": "{{ dep }}"}, + "loop": ["role-b", "role-c"], + "loop_control": {"loop_var": "dep"}, + } + ] + self.assertEqual(self._invoked(tasks), {"role-b", "role-c"}) + + def test_a_template_with_a_literal_part_matches_a_glob(self): + tasks = [{"include_role": {"name": "sys-ctl-hlth-{{ kind }}"}}] + self.assertEqual(self._invoked(tasks), {"sys-ctl-hlth-x", "sys-ctl-hlth-y"}) + + def test_a_role_name_handed_over_as_a_var_is_an_edge(self): + tasks = [ + { + "include_tasks": "utils/load/app.yml", + "vars": {"load_app_id": "role-b"}, + } + ] + self.assertEqual(self._invoked(tasks), {"role-b"}) + + def test_including_a_task_file_of_another_role_is_an_edge(self): + tasks = [ + { + "include_tasks": "{{ lookup('path_absolute', 'roles/role-c/tasks/x.yml') }}" + } + ] + self.assertEqual(self._invoked(tasks), {"role-c"}) + + def test_includes_nested_in_blocks_count(self): + tasks = [ + { + "block": [{"include_role": {"name": "role-b"}}], + "rescue": [{"include_role": {"name": "role-c"}}], + } + ] + self.assertEqual(self._invoked(tasks), {"role-b", "role-c"}) + + def test_conditions_do_not_hide_an_edge(self): + tasks = [{"include_role": {"name": "role-b"}, "when": "false"}] + self.assertEqual(self._invoked(tasks), {"role-b"}) + + def test_a_bare_template_falls_back_to_quoted_role_names_in_vars(self): + self._write( + _role_file("role-a", ROLE_FILE_VARS_MAIN), + {"DEPS": "{{ ['role-b'] + (['role-c'] if flag else []) }}"}, + ) + tasks = [ + { + "include_role": {"name": "{{ dep }}"}, + "loop": "{{ DEPS }}", + "loop_control": {"loop_var": "dep"}, + } + ] + self.assertEqual(self._invoked(tasks), {"role-b", "role-c"}) + + def test_quoted_names_are_ignored_while_every_include_resolves(self): + self._write(_role_file("role-a", ROLE_FILE_VARS_MAIN), {"OTHER": "{{ 'role-c' }}"}) + self.assertEqual( + self._invoked([{"include_role": {"name": "role-b"}}]), {"role-b"} + ) + + def test_unknown_role_names_are_dropped(self): + self.assertEqual( + self._invoked([{"include_role": {"name": "role-missing"}}]), frozenset() + ) + + def test_a_role_does_not_invoke_itself(self): + self.assertEqual( + self._invoked([{"include_role": {"name": "role-a"}}]), frozenset() + ) + + def test_handlers_count(self): + self._write( + _role_file("role-a", ROLE_FILE_HANDLERS_MAIN), + [{"include_role": {"name": "role-c"}}], + ) + self.assertEqual( + self._invoked([{"include_role": {"name": "role-b"}}]), {"role-b", "role-c"} + ) + + +class TestStageInvocations(_TreeMixin, unittest.TestCase): + def _stage(self, tasks): + return self._write("tasks/stages/01_constructor.yml", tasks) + + def test_a_group_membership_gate_closes_for_a_non_member(self): + path = self._stage( + [{"include_role": {"name": "role-b"}, "when": "'role-b' in group_names"}] + ) + self.assertEqual( + stage_invocations(path, frozenset(ROLES), seeds=[], group_names=[]), set() + ) + + def test_a_group_membership_gate_opens_for_a_member(self): + path = self._stage( + [{"include_role": {"name": "role-b"}, "when": "'role-b' in group_names"}] + ) + self.assertEqual( + stage_invocations( + path, frozenset(ROLES), seeds=[], group_names=["role-b"] + ), + {"role-b"}, + ) + + def test_an_application_allowed_gate_decides_on_the_seeds(self): + path = self._stage( + [ + { + "when": "'role-b' | application_allowed(group_names, whitelist)", + "block": [{"include_role": {"name": "role-b"}}], + } + ] + ) + self.assertEqual( + stage_invocations( + path, frozenset(ROLES), seeds=[], group_names=["role-b"] + ), + set(), + ) + self.assertEqual( + stage_invocations( + path, frozenset(ROLES), seeds=["role-b"], group_names=[] + ), + {"role-b"}, + ) + + def test_any_other_condition_keeps_the_include(self): + path = self._stage( + [{"include_role": {"name": "role-c"}, "when": ["MODE_CLEANUP | bool"]}] + ) + self.assertEqual( + stage_invocations(path, frozenset(ROLES), seeds=[], group_names=[]), + {"role-c"}, + ) + + def test_stage_files_follow_the_host_types(self): + for name in ("01_constructor", "02_server", "02_workstation", "03_destructor"): + self._write(f"tasks/stages/{name}.yml", []) + self.assertEqual( + [path.name for path in stage_files(self.root, ["server"])], + ["01_constructor.yml", "02_server.yml", "03_destructor.yml"], + ) + + +class TestClosure(_TreeMixin, unittest.TestCase): + def test_invocations_are_followed_transitively(self): + self._write( + _role_file("role-b", ROLE_FILE_TASKS_MAIN), + [{"include_role": {"name": "role-c"}}], + ) + self._write( + _role_file("role-a", ROLE_FILE_TASKS_MAIN), + [{"include_role": {"name": "role-b"}}], + ) + self.assertEqual( + expand_roles({}, ["role-a"], project_root=self.root), + {"role-a", "role-b", "role-c"}, + ) + + def test_meta_dependencies_are_followed(self): + self._write( + _role_file("role-a", ROLE_FILE_META_MAIN), {"dependencies": ["role-c"]} + ) + self.assertEqual( + expand_roles({}, ["role-a"], project_root=self.root), {"role-a", "role-c"} + ) + + def test_the_deploy_closure_adds_the_stage_roles(self): + self._write( + "tasks/stages/01_constructor.yml", [{"include_role": {"name": "role-c"}}] + ) + self.assertEqual( + deploy_closure( + {}, + ["role-a"], + project_root=self.root, + host_types=[], + group_names=["role-a"], + ), + {"role-a", "role-c"}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/packages/batch.py b/utils/packages/batch.py new file mode 100644 index 0000000000..70f820d0c1 --- /dev/null +++ b/utils/packages/batch.py @@ -0,0 +1,48 @@ +"""The package ids a deploy installs in one run at its start. + +A role's batched ids are the ids its own ``meta/packages.yml`` declares plus +the shared root ids it references there, minus every id flagged +``batch: false``. Which roles a deploy may run is +:func:`utils.roles.invocations.deploy_closure`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from utils.packages.registry import build_registry, load_references + +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + + +def batch_ids_by_role(project_root: Path) -> dict[str, set[str]]: + """Return, per role, the ids the deploy-start batch installs for it. + + Args: + project_root: repository root holding the package registry. + """ + registry = build_registry(project_root) + by_role: dict[str, set[str]] = {} + for package_id, declaration in registry.items(): + if declaration.role is not None and declaration.batch: + by_role.setdefault(declaration.role, set()).add(package_id) + for role, package_ids in load_references(project_root).items(): + by_role.setdefault(role, set()).update( + package_id + for package_id in package_ids + if package_id in registry and registry[package_id].batch + ) + return by_role + + +def batch_ids(roles: Iterable[str], project_root: Path) -> list[str]: + """Return the sorted ids the deploy-start batch installs for *roles*. + + Args: + roles: every role the deploy may run on the host. + project_root: repository root holding the package registry. + """ + by_role = batch_ids_by_role(project_root) + return sorted({package_id for role in roles for package_id in by_role.get(role, ())}) diff --git a/utils/packages/plan.py b/utils/packages/plan.py index 04eed01033..c047cb9c95 100644 --- a/utils/packages/plan.py +++ b/utils/packages/plan.py @@ -9,7 +9,10 @@ executes what it is handed. from __future__ import annotations +from dataclasses import replace + from utils.packages.calls import ( + GENERIC_PACKAGE, STATE_ABSENT, STATE_PRESENT, ModuleCall, @@ -48,3 +51,49 @@ def build_plan(spec: PackageSpec, state: str) -> list[ModuleCall]: if spec.repo: return repository_calls(spec.repo, names) return [package_call(names, STATE_PRESENT)] + + +def _plain(plan: list[ModuleCall]) -> bool: + if not plan or any( + call.module != GENERIC_PACKAGE + or call.become_user is not None + or set(call.args) != {"name", "state"} + for call in plan + ): + return False + return len({call.args["state"] for call in plan}) == 1 + + +def merge_plans(plans: list[list[ModuleCall]]) -> list[list[ModuleCall]]: + """Fold the plans that only call the host's package manager into one plan. + + The plain plans are aligned on their last call, so the n-th call from + the end of every plan lands in the same merged call and each plan's own + order survives: a bootstrap package still installs before the names it + unlocks. Every other plan follows unchanged, in request order. + + Args: + plans: one plan per package id, in request order. + + Returns: + one merged plan per state the plain plans bring their names into, + then every other plan. + """ + groups: dict[str, list[list[ModuleCall]]] = {} + for plan in plans: + if _plain(plan): + groups.setdefault(plan[-1].args["state"], []).append(plan) + return [_merge(group) for group in groups.values()] + [ + plan for plan in plans if not _plain(plan) + ] + + +def _merge(plain: list[list[ModuleCall]]) -> list[ModuleCall]: + merged: list[ModuleCall] = [] + for offset in range(max(len(plan) for plan in plain), 0, -1): + calls = [plan[-offset] for plan in plain if len(plan) >= offset] + names = list( + dict.fromkeys(name for call in calls for name in call.args["name"]) + ) + merged.append(replace(calls[0], args={**calls[0].args, "name": names})) + return merged diff --git a/utils/packages/registry.py b/utils/packages/registry.py index d2dac39382..10098914fd 100644 --- a/utils/packages/registry.py +++ b/utils/packages/registry.py @@ -25,9 +25,11 @@ if TYPE_CHECKING: from utils.cache import PROJECT_ROOT from utils.cache.yaml import load_yaml_any from utils.packages.schema import ( + BATCH_KEY, DISTRO_FAMILY, INVENTORY_PACKAGES_VAR, ROLE_FILE_META_PACKAGES, + SHARED_REFERENCE, PackageSpec, PackagesShapeError, normalize_entry, @@ -53,6 +55,11 @@ class Declaration: """Human-readable owner for lint and runtime messages.""" return self.role or "the shared registry" + @property + def batch(self) -> bool: + """Whether the deploy-start batch installs this id.""" + return self.mapping.get(BATCH_KEY, True) + def iter_package_files(project_root: Path) -> Iterator[tuple[str | None, Path]]: """Yield ``(role_or_None, path)`` for every package declaration file. @@ -73,13 +80,7 @@ def iter_package_files(project_root: Path) -> Iterator[tuple[str | None, Path]]: yield role_dir.name, path -def load_declarations(project_root: Path) -> list[Declaration]: - """Read every ``meta/packages.yml`` without deduplicating. - - Duplicates are returned as-is so the uniqueness lint can report all - offenders instead of failing on the first one. - """ - declarations: list[Declaration] = [] +def _entries(project_root: Path) -> Iterator[tuple[str | None, Path, str, Any]]: for role, path in iter_package_files(project_root): raw = load_yaml_any(str(path), default_if_missing={}) or {} if not isinstance(raw, dict): @@ -88,15 +89,46 @@ def load_declarations(project_root: Path) -> list[Declaration]: f"got {type(raw).__name__}." ) for package_id, mapping in raw.items(): - if not isinstance(mapping, dict): - raise PackagesShapeError( - f"{path}: package '{package_id}' must map distro keys to " - f"specs, got {type(mapping).__name__}." - ) - declarations.append(Declaration(str(package_id), role, path, mapping)) + yield role, path, str(package_id), mapping + + +def _is_reference(role: str | None, mapping: Any) -> bool: + return role is not None and mapping == SHARED_REFERENCE + + +def load_declarations(project_root: Path) -> list[Declaration]: + """Read every ``meta/packages.yml`` without deduplicating. + + Duplicates are returned as-is so the uniqueness lint can report all + offenders instead of failing on the first one. A role's reference to a + shared id is no declaration and is left to :func:`load_references`. + """ + declarations: list[Declaration] = [] + for role, path, package_id, mapping in _entries(project_root): + if _is_reference(role, mapping): + continue + if not isinstance(mapping, dict): + raise PackagesShapeError( + f"{path}: package '{package_id}' must map distro keys to " + f"specs, got {type(mapping).__name__}." + ) + if not isinstance(mapping.get(BATCH_KEY, False), bool): + raise PackagesShapeError( + f"{path}: {package_id}.{BATCH_KEY} must be a boolean when set." + ) + declarations.append(Declaration(package_id, role, path, mapping)) return declarations +def load_references(project_root: Path) -> dict[str, set[str]]: + """Return, per role, the shared root ids its own file references.""" + references: dict[str, set[str]] = {} + for role, _path, package_id, mapping in _entries(project_root): + if _is_reference(role, mapping): + references.setdefault(str(role), set()).add(package_id) + return references + + def iter_inventory_package_lists( project_root: Path, ) -> Iterator[tuple[Path, list[str]]]: diff --git a/utils/packages/schema.py b/utils/packages/schema.py index 18e5dddcc1..f38c25f036 100644 --- a/utils/packages/schema.py +++ b/utils/packages/schema.py @@ -41,6 +41,26 @@ which counts as covered; a *missing* key is a coverage gap. ``virtual: true`` records that the package manager satisfies the name through an RPM ``Provides`` rather than a package of that name, so an index lookup by name legitimately finds nothing while the install works. + +Every id is installed by the one ``package_install`` run the constructor +starts each deploy with, for every role that deploy may run. ``batch: false`` +on the id, next to its distro keys, takes it out of that run; the role's own +``package_install`` task then installs it, which is required when the install +is conditional or must wait for a setup step:: + + docker-cli: + Archlinux: [docker] + Debian: + names: [docker-ce] + repo: + managed_externally: the role adds the Docker repository first + batch: false + +A role that needs an id of the shared root file references it in its own +file, so the batch knows to install it with the role:: + + git: + shared: true """ from __future__ import annotations @@ -57,6 +77,12 @@ ROLE_FILE_META_PACKAGES = "meta/packages.yml" INVENTORY_PACKAGES_VAR = "PACKAGES" """Inventory variable listing extra package ids a bundle installs.""" +BATCH_KEY = "batch" +"""Id-level flag; ``false`` keeps the id out of the deploy-start batch.""" + +SHARED_REFERENCE: dict[str, bool] = {"shared": True} +"""The whole mapping of a role-file entry that references a root id.""" + SOURCE_REPO = "repo" SOURCE_AUR = "aur" SOURCE_COPR = "copr" diff --git a/utils/roles/invocations.py b/utils/roles/invocations.py new file mode 100644 index 0000000000..ad52eb1674 --- /dev/null +++ b/utils/roles/invocations.py @@ -0,0 +1,326 @@ +"""Roles a deploy may run, read statically from task files. + +A role invokes another role when an ``include_role``/``import_role`` names it +(literally, through a literal loop item, or through a template whose literal +part narrows a glob), when one of its tasks hands a role name to an include +as a ``vars`` value (``load_app_id``, ``list_or_shoot_roles``), and when it +includes a task or handler file of that role by path. A role that includes +a bare template no loop resolves invokes every role its ``vars`` and +``defaults`` name in quotes. + +Conditions inside roles are ignored, so :func:`deploy_closure` returns every +role a deploy may run, a superset of the roles it will run. The play's stage +files are the one exception: a group-membership gate there +(``'' in group_names``, ``'' | application_allowed(...)``) is +decided against the host, because it is what selects the host's roles. +""" + +from __future__ import annotations + +import fnmatch +import re +from functools import cache +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from utils.cache.files import read_text +from utils.cache.yaml import load_yaml_any +from utils.roles.applications.in_group_deps import reachable_roles + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + +INCLUDE_ROLE_KEYS: frozenset[str] = frozenset( + { + "include_role", + "import_role", + "ansible.builtin.include_role", + "ansible.builtin.import_role", + } +) +TASK_FILE_DIRS: tuple[str, ...] = ("tasks", "handlers") +TEMPLATE_DIRS: tuple[str, ...] = ("vars", "defaults") +LOOP_KEYS: tuple[str, ...] = ("loop", "with_items", "with_list") +NESTED_KEYS: tuple[str, ...] = ("block", "rescue", "always") +STAGES_DIR = Path("tasks") / "stages" +CONSTRUCTOR_STAGE = "01_constructor.yml" +DESTRUCTOR_STAGE = "03_destructor.yml" + +_ROLE_FILE_REF_RE = re.compile(r"roles/(?P[a-z0-9-]+)/(?:tasks|handlers)/") +_QUOTED_NAME_RE = re.compile(r"['\"](?P[a-z0-9][a-z0-9-]*)['\"]") +_TEMPLATE_RE = re.compile(r"\{\{.*?\}\}") +_MEMBERSHIP_GATE_RE = re.compile( + r"^'(?P[a-z0-9-]+)'\s*(?Pin\s+group_names|\|\s*application_allowed\b)" +) + + +@cache +def role_names(roles_dir: str) -> frozenset[str]: + """Return the name of every role directory under *roles_dir*.""" + root = Path(roles_dir) + if not root.is_dir(): + return frozenset() + return frozenset(path.name for path in root.iterdir() if path.is_dir()) + + +def iter_tasks(node: Any) -> Iterator[dict[str, Any]]: + """Yield every task of a task list, descending into blocks.""" + if not isinstance(node, list): + return + for task in node: + if not isinstance(task, dict): + continue + yield task + for key in NESTED_KEYS: + yield from iter_tasks(task.get(key)) + + +def _strings(value: Any) -> Iterator[str]: + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for item in value.values(): + yield from _strings(item) + elif isinstance(value, list): + for item in value: + yield from _strings(item) + + +def _named(name: str, known: frozenset[str]) -> set[str] | None: + name = name.strip() + if "{{" not in name: + return {name} + pattern = _TEMPLATE_RE.sub("*", name) + if not pattern.strip("*"): + return None + return set(fnmatch.filter(known, pattern)) + + +def _looped(task: dict[str, Any]) -> set[str] | None: + for key in LOOP_KEYS: + if key not in task: + continue + items = task[key] + if not isinstance(items, list): + return None + found: set[str] = set() + for item in items: + if isinstance(item, str): + found.add(item.strip()) + elif isinstance(item, dict): + found.update( + str(item[field]).strip() + for field in ("role", "name") + if isinstance(item.get(field), str) + ) + return found + return None + + +def task_invocations( + task: dict[str, Any], known: frozenset[str] +) -> tuple[set[str], bool]: + """Return the roles one task invokes, and whether a name stayed unresolved. + + Args: + task: one task mapping; the tasks nested in its blocks are not read. + known: every existing role name. + """ + own = {key: value for key, value in task.items() if key not in NESTED_KEYS} + invoked = {value.strip() for value in _strings(own.get("vars"))} & known + for value in _strings(own): + invoked.update( + match.group("role") for match in _ROLE_FILE_REF_RE.finditer(value) + ) + + unresolved = False + for key in INCLUDE_ROLE_KEYS & own.keys(): + spec = own[key] + name = spec.get("name") if isinstance(spec, dict) else spec + named = _named(name, known) if isinstance(name, str) else None + if named is None: + named = _looped(own) + if named is None: + unresolved = True + continue + invoked |= named + return invoked & known, unresolved + + +def file_invocations(path: Path, known: frozenset[str]) -> tuple[set[str], bool]: + """Return the roles one task file invokes, and whether a name stayed unresolved. + + Args: + path: a task or handler file. + known: every existing role name. + """ + invoked: set[str] = set() + unresolved = False + for task in iter_tasks(load_yaml_any(str(path), default_if_missing=None)): + roles, open_name = task_invocations(task, known) + invoked |= roles + unresolved = unresolved or open_name + return invoked, unresolved + + +def _files(role_dir: Path, subdirs: Iterable[str]) -> list[Path]: + return sorted( + path + for sub in subdirs + if (role_dir / sub).is_dir() + for path in (role_dir / sub).rglob("*.yml") + ) + + +@cache +def role_invocations(roles_dir: str, role: str) -> frozenset[str]: + """Return every other role *role* may invoke. + + Args: + roles_dir: directory holding the role folders. + role: role name. + """ + known = role_names(roles_dir) + role_dir = Path(roles_dir) / role + invoked: set[str] = set() + unresolved = False + for path in _files(role_dir, TASK_FILE_DIRS): + roles, open_name = file_invocations(path, known) + invoked |= roles + unresolved = unresolved or open_name + if unresolved: + for path in _files(role_dir, TEMPLATE_DIRS): + invoked.update( + match.group("name") + for match in _QUOTED_NAME_RE.finditer(read_text(str(path))) + ) + invoked.discard(role) + return frozenset(invoked & known) + + +def stage_files(project_root: Path, host_types: Iterable[str]) -> list[Path]: + """Return the play's stage files a host with *host_types* runs, in order. + + Args: + project_root: repository root. + host_types: the host's deployment types (``server``, ``workstation``). + """ + stages = Path(project_root) / STAGES_DIR + names = [ + CONSTRUCTOR_STAGE, + *(f"02_{host_type}.yml" for host_type in host_types), + DESTRUCTOR_STAGE, + ] + return [stages / name for name in names if (stages / name).is_file()] + + +def _gate_closed( + task: dict[str, Any], seeds: frozenset[str], group_names: frozenset[str] +) -> bool: + when = task.get("when") + for condition in when if isinstance(when, list) else [when]: + if not isinstance(condition, str): + continue + match = _MEMBERSHIP_GATE_RE.match(condition.strip()) + if match is None: + continue + members = group_names if match.group("test").startswith("in") else seeds + if match.group("role") not in members: + return True + return False + + +def _open_tasks( + node: Any, seeds: frozenset[str], group_names: frozenset[str] +) -> Iterator[dict[str, Any]]: + if not isinstance(node, list): + return + for task in node: + if not isinstance(task, dict) or _gate_closed(task, seeds, group_names): + continue + yield task + for key in NESTED_KEYS: + yield from _open_tasks(task.get(key), seeds, group_names) + + +def stage_invocations( + path: Path, + known: frozenset[str], + *, + seeds: Iterable[str], + group_names: Iterable[str], +) -> set[str]: + """Return the roles a stage file invokes on one host. + + Args: + path: a file under ``tasks/stages/``. + known: every existing role name. + seeds: the application roles the host deploys. + group_names: the host's inventory groups. + """ + doc = load_yaml_any(str(path), default_if_missing=None) + invoked: set[str] = set() + for task in _open_tasks(doc, frozenset(seeds), frozenset(group_names)): + invoked |= task_invocations(task, known)[0] + return invoked + + +def deploy_closure( + applications: dict[str, Any], + seeds: Iterable[str], + *, + project_root: Path, + host_types: Iterable[str], + group_names: Iterable[str], +) -> set[str]: + """Return every role a deploy may run on one host. + + Args: + applications: the merged applications map, for service edges. + seeds: the application roles the host deploys. + project_root: repository root. + host_types: the host's deployment types, selecting its stage files. + group_names: the host's inventory groups. + """ + known = role_names(str(Path(project_root) / "roles")) + seed_set = {str(seed) for seed in seeds} + roots = set(seed_set) + for path in stage_files(project_root, host_types): + roots |= stage_invocations(path, known, seeds=seed_set, group_names=group_names) + return expand_roles(applications, roots, project_root=project_root) + + +def expand_roles( + applications: dict[str, Any], roles: Iterable[str], *, project_root: Path +) -> set[str]: + """Return *roles* plus every role they reach, transitively. + + Edges are ``meta/main.yml`` dependencies, the service edges of + *applications*, and :func:`role_invocations`. + + Args: + applications: the merged applications map; empty for no service edges. + roles: the roles to start from. + project_root: repository root. + """ + roles_dir = str(Path(project_root) / "roles") + known = role_names(roles_dir) + frontier = {str(role) for role in roles} + reached: set[str] = set() + while frontier: + expanded = ( + reachable_roles( + applications, + sorted(frontier), + project_root=str(project_root), + roles_dir=roles_dir, + ) + | frontier + ) - reached + reached |= expanded + frontier = { + invoked + for role in expanded & known + for invoked in role_invocations(roles_dir, role) + } - reached + return reached