From 70ed8a2e983c0983608efd6433e8e66dd828d7ba Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 29 Oct 2026 09:35:12 +0100 Subject: [PATCH] fix(service-loader): per-host post-load queue, Docker timers queued by sys-svc-container sys-stk-full includes sys-stk-front-proxy with apply run_once, so the set_fact in list_or_shoot.yml that appends to service_loader_post_run inherited run_once. ansible-core writes a run_once set_fact to every play host, rendered with the first host's vars, so the stack host's whole queue replaced every other host's queue: in run3 the workers and the NFS server flushed that copy, re-ran web-app-stalwart's core and the Docker timer roles, and the NFS server got Docker that way. An explicit run_once: false overrides the inherited value and keeps the queue on the host that built it. Swarm workers had sys-ctl-cln-docker, sys-ctl-hlth-container and sys-ctl-hlth-volumes only through that leak, because sys-svc-compose queued them on hosts that run compose projects. The owner of the Docker daemon now queues them: sys-svc-container routes the three roles through list_or_shoot at the end of its run-once core, so every host that installs Docker gets them once and hosts without Docker, like the NFS server and the backup host, get none. sys-svc-compose keeps sys-ctl-rpr-container-hard, which repairs compose projects, and svc-bkp-volume-2-local. The constructor now opens the preload context before the svc-swarm-node bootstrap, so the Docker install on swarm nodes queues the timers instead of running them before the token store and the mail provider exist, and the context reset no longer wipes them. The loader flushes them after the shared services. The loader now clears service_loader_preloading right before its post-load flush, as its header documents. It only cleared the flag per loaded service, so a host that preloads no shared service, a swarm worker, flushed with the flag still set: every list_or_shoot call made during the flush was queued after the flush loop had been evaluated and lost, and the workers' alarm path would never have installed sys-svc-mail, its msmtp client and sys-ctl-hlth-msmtp. list_or_shoot now includes a single role without a loop. The alarm path re-enters it (sys-ctl-alm-email routes sys-svc-mail, whose msmtp role routes sys-ctl-hlth-msmtp), and a list_or_shoot loop running inside another one makes ansible-core warn that the variable 'list_or_shoot_role' is already in use. Every call that nests routes a single role, and ansible-core checks the loop variable only for loops with items, so that chain now runs without the warning. A scratch playbook drove the real list_or_shoot.yml, the loader flush, the reordered constructor tasks and the routes of sys-svc-container and sys-svc-compose, extracted verbatim, with recorder roles for the timers and the alarm path. On ansible-core 2.20.6 and 2.21.5 it matched the expected per-host run sets for swarm, backup-only, backup with Docker, compose via pre-pull, compose via sys-svc-compose, whitelist, swarm without web apps and a Docker workstation, with no "already in use" warning. Opening the context after the bootstrap ran the timers before the token store, flushing without clearing the flag lost the workers' mail path, and the looped single-role shoot logged the warning again. Co-Authored-By: Claude Opus 5.5 (1M context) --- roles/sys-service-loader/tasks/list_or_shoot.yml | 12 +++++++++++- roles/sys-service-loader/tasks/main.yml | 5 +++++ roles/sys-svc-compose/tasks/main.yml | 5 +---- roles/sys-svc-container/tasks/00_core.yml | 10 ++++++++++ tasks/stages/01_constructor.yml | 10 +++++----- 5 files changed, 32 insertions(+), 10 deletions(-) diff --git a/roles/sys-service-loader/tasks/list_or_shoot.yml b/roles/sys-service-loader/tasks/list_or_shoot.yml index 93988bd020..08fab0241e 100644 --- a/roles/sys-service-loader/tasks/list_or_shoot.yml +++ b/roles/sys-service-loader/tasks/list_or_shoot.yml @@ -24,14 +24,24 @@ ansible.builtin.set_fact: service_loader_post_run: "{{ (service_loader_post_run | default([])) + list_or_shoot_roles }}" changed_when: false + run_once: false when: (service_loader_preloading | default(false) | bool) +- name: "🎯 Shoot the post-load role directly (no preload context active)" + ansible.builtin.include_role: + name: "{{ list_or_shoot_roles | first }}" + vars: + application_id: "{{ list_or_shoot_roles | first }}" + when: + - not (service_loader_preloading | default(false) | bool) + - list_or_shoot_roles | length == 1 + - name: "🎯 Shoot post-load roles directly (no preload context active)" ansible.builtin.include_role: name: "{{ list_or_shoot_role }}" vars: application_id: "{{ list_or_shoot_role }}" - loop: "{{ list_or_shoot_roles }}" + loop: "{{ list_or_shoot_roles if list_or_shoot_roles | length > 1 else [] }}" loop_control: loop_var: list_or_shoot_role when: not (service_loader_preloading | default(false) | bool) diff --git a/roles/sys-service-loader/tasks/main.yml b/roles/sys-service-loader/tasks/main.yml index b7360c5d40..feb6b6de9c 100644 --- a/roles/sys-service-loader/tasks/main.yml +++ b/roles/sys-service-loader/tasks/main.yml @@ -55,6 +55,11 @@ loop_var: service_item label: "{{ service_item.id }}" +- name: "📂 Close the preload context before the post-load flush" + ansible.builtin.set_fact: + service_loader_preloading: false + changed_when: false + - name: "▶️ Run service-loader post-load roles" ansible.builtin.include_role: name: "{{ post_run_role }}" diff --git a/roles/sys-svc-compose/tasks/main.yml b/roles/sys-svc-compose/tasks/main.yml index 6a69e41198..af0c5c4153 100644 --- a/roles/sys-svc-compose/tasks/main.yml +++ b/roles/sys-svc-compose/tasks/main.yml @@ -2,15 +2,12 @@ - include_tasks: 00_core.yml when: run_once_sys_svc_compose is not defined -- name: "🧭 Route docker cleanup/health deps through service-loader" +- name: "🧭 Route the compose hard repair through service-loader" ansible.builtin.include_role: name: sys-service-loader tasks_from: list_or_shoot.yml vars: list_or_shoot_roles: - - sys-ctl-cln-docker - - sys-ctl-hlth-container - - sys-ctl-hlth-volumes - sys-ctl-rpr-container-hard - name: "🧭 Route svc-bkp-volume-2-local through service-loader (when deployed)" diff --git a/roles/sys-svc-container/tasks/00_core.yml b/roles/sys-svc-container/tasks/00_core.yml index 1acdc12f73..d715d4da06 100644 --- a/roles/sys-svc-container/tasks/00_core.yml +++ b/roles/sys-svc-container/tasks/00_core.yml @@ -94,4 +94,14 @@ include_tasks: "03_reset.yml" when: MODE_RESET | bool +- name: "🧭 Route docker cleanup/health deps through service-loader" + ansible.builtin.include_role: + name: sys-service-loader + tasks_from: list_or_shoot.yml + vars: + list_or_shoot_roles: + - sys-ctl-cln-docker + - sys-ctl-hlth-container + - sys-ctl-hlth-volumes + - include_tasks: utils/once/flag.yml diff --git a/tasks/stages/01_constructor.yml b/tasks/stages/01_constructor.yml index bd21b7648f..dc6b768cfc 100644 --- a/tasks/stages/01_constructor.yml +++ b/tasks/stages/01_constructor.yml @@ -191,17 +191,17 @@ }} when: _package_batch_ids | length > 0 -- name: "🥾 Bootstrap svc-swarm-node" - ansible.builtin.include_role: - name: svc-swarm-node - when: "'svc-swarm-node' | application_allowed(group_names, lookup('deployment').whitelist)" - - name: "📂 Open service-loader preload context BEFORE Bootstrap" ansible.builtin.set_fact: service_loader_post_run: [] service_loader_preloading: true changed_when: false +- name: "🥾 Bootstrap svc-swarm-node" + ansible.builtin.include_role: + name: svc-swarm-node + when: "'svc-swarm-node' | application_allowed(group_names, lookup('deployment').whitelist)" + - name: "🥾 Bootstrap NFS clients + registry + sandbox runtimes (MUST precede Preload)" ansible.builtin.include_tasks: ./tasks/utils/setup/group.yml loop: