perf(spot): carry the rendered users next to the applications

Ansible forks a fresh worker per task, so every lookup('users') in a forked worker re-rendered the whole user catalogue (298 users, 0.3 to 0.5 s locally and about 1 s on the runner, 1 s locally once a users password is vaulted): _MERGED_USERS_CACHE lives in the process and had no carrier fact.

applications_carrier now also parks lookup('users', carrier=True) in _INFINITO_USERS_RENDERED at the same park points, and clear drops both facts. get_merged_users serves the fact to templated reads when its key matches: the applications carrier key (roles dir, applications, users, DOMAIN_PRIMARY, SYSTEM_EMAIL_DOMAIN, group_names, token store digest) plus inventory_hostname, so a carrier a run_once park broadcasts never answers another host with its become password. A park whose users carrier still matches leaves it in place instead of returning 119 KB again, so parks cost what they cost before.

Real ansible-playbook harness on 2.20.6 and 2.21.5 with the real roles tree: users renders per users-reading task after a park 1 -> 0, task 0.56 -> 0.08 s (vaulted 1.21 -> 0.17 s); a rotated token, a rotated inventory password, a cleared carrier and another host's carrier all re-render; 32 of 32 probe digests equal HEAD's. Removing the carrier check fails three unit tests. The stalwart swarm roundtrip pass 1 and its DR drill ran green with it on all nodes, with no 401 from JMAP, msmtp or LDAP.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

tests/unit/python/plugins/action/test_applications_carrier.py

@@ -4,7 +4,7 @@ from unittest.mock import patch
 from ansible.errors import AnsibleActionFail
 
 from plugins.action.applications_carrier import ActionModule
-from utils.cache.carrier import APPLICATIONS_RENDERED_FACT
+from utils.cache.carrier import APPLICATIONS_RENDERED_FACT, USERS_RENDERED_FACT
 
 
 class _FakeTask:
@@ -31,6 +31,16 @@ class _FakeLookup:
         return [self.carrier]
 
 
+class _FakeLoader:
+    def __init__(self, applications, users):
+        self.lookups = {"applications": applications, "users": users}
+        self.order = []
+
+    def get(self, name, loader=None, templar=None):
+        self.order.append(name)
+        return self.lookups[name]
+
+
 def _make_action(task: _FakeTask, templar=None) -> ActionModule:
     action = object.__new__(ActionModule)
     action._task = task
@@ -39,10 +49,19 @@ def _make_action(task: _FakeTask, templar=None) -> ActionModule:
     return action
 
 
+def _lookups(templar=None):
+    applications = _FakeLookup(
+        {"key": ["roles", ["a", "b", "c", "d"]], "applications": {"x": {}}}, templar
+    )
+    users = _FakeLookup(
+        {"key": ["roles", ["a", "b", "c", "d", "h"]], "users": {"u": {}}}, templar
+    )
+    return applications, users, _FakeLoader(applications, users)
+
+
 class TestApplicationsCarrierActionPlugin(unittest.TestCase):
-    def test_parks_the_carrier_as_a_masked_fact(self):
-        carrier = {"key": ["roles", ["a", "b", "c", "d"]], "applications": {"x": {}}}
-        lookup = _FakeLookup(carrier)
+    def test_parks_both_carriers_as_masked_facts(self):
+        applications, users, loader = _lookups()
         action = _make_action(_FakeTask())
         with (
             patch(
@@ -52,23 +71,78 @@ class TestApplicationsCarrierActionPlugin(unittest.TestCase):
             ),
             patch(
                 "plugins.action.applications_carrier.lookup_loader.get",
-                return_value=lookup,
+                side_effect=loader.get,
             ),
         ):
             result = action.run(task_vars={"applications": {}})
 
-        self.assertIs(result["ansible_facts"][APPLICATIONS_RENDERED_FACT], carrier)
+        self.assertIs(
+            result["ansible_facts"][APPLICATIONS_RENDERED_FACT], applications.carrier
+        )
+        self.assertIs(result["ansible_facts"][USERS_RENDERED_FACT], users.carrier)
         self.assertTrue(result["_ansible_no_log"])
         self.assertFalse(result["changed"])
-        terms, variables, kwargs = lookup.calls[0]
-        self.assertEqual(terms, [])
-        self.assertEqual(variables, {"applications": {}})
-        self.assertTrue(kwargs["carrier"])
+        for lookup in (applications, users):
+            terms, variables, kwargs = lookup.calls[0]
+            self.assertEqual(terms, [])
+            self.assertEqual(variables, {"applications": {}})
+            self.assertTrue(kwargs["carrier"])
+
+    def _park(self, task_vars):
+        applications, users, loader = _lookups()
+        action = _make_action(_FakeTask())
+        with (
+            patch(
+                "plugins.action.applications_carrier.ActionBase.run",
+                autospec=True,
+                return_value={},
+            ),
+            patch(
+                "plugins.action.applications_carrier.lookup_loader.get",
+                side_effect=loader.get,
+            ),
+            patch("plugins.action.applications_carrier.revalidate_tokens_file"),
+        ):
+            return action.run(task_vars=task_vars), applications, users
+
+    def test_a_users_carrier_that_still_matches_stays_in_place(self):
+        parked = {"key": ["roles", ["a", "b", "c", "d", "h"]], "users": {"u": {}}}
+        result, applications, _ = self._park({USERS_RENDERED_FACT: parked})
+
+        self.assertNotIn(USERS_RENDERED_FACT, result["ansible_facts"])
+        self.assertIs(
+            result["ansible_facts"][APPLICATIONS_RENDERED_FACT], applications.carrier
+        )
+
+    def test_a_stale_users_carrier_is_replaced(self):
+        parked = {"key": ["roles", ["a", "b", "c", "d", "h2"]], "users": {"u": {}}}
+        result, _, users = self._park({USERS_RENDERED_FACT: parked})
+
+        self.assertIs(result["ansible_facts"][USERS_RENDERED_FACT], users.carrier)
+
+    def test_renders_the_applications_before_the_users(self):
+        _, _, loader = _lookups()
+        action = _make_action(_FakeTask())
+        with (
+            patch(
+                "plugins.action.applications_carrier.ActionBase.run",
+                autospec=True,
+                return_value={},
+            ),
+            patch(
+                "plugins.action.applications_carrier.lookup_loader.get",
+                side_effect=loader.get,
+            ),
+            patch("plugins.action.applications_carrier.revalidate_tokens_file"),
+        ):
+            action.run(task_vars={})
+
+        self.assertEqual(loader.order, ["applications", "users"])
 
     def test_renders_without_the_callers_application_id(self):
         task_vars = {"applications": {}, "application_id": "web-app-bluesky"}
         templar = _FakeTemplar(task_vars)
-        lookup = _FakeLookup({"key": ["roles", []], "applications": {}}, templar)
+        applications, users, loader = _lookups(templar)
         action = _make_action(_FakeTask(), templar)
         with (
             patch(
@@ -78,22 +152,26 @@ class TestApplicationsCarrierActionPlugin(unittest.TestCase):
             ),
             patch(
                 "plugins.action.applications_carrier.lookup_loader.get",
-                return_value=lookup,
+                side_effect=loader.get,
             ),
             patch("plugins.action.applications_carrier.revalidate_tokens_file"),
         ):
             action.run(task_vars=task_vars)
 
-        _, variables, _ = lookup.calls[0]
-        self.assertEqual(variables, {"applications": {}})
-        self.assertEqual(lookup.templar_variables, [{"applications": {}}])
+        for lookup in (applications, users):
+            _, variables, _ = lookup.calls[0]
+            self.assertEqual(variables, {"applications": {}})
+            self.assertEqual(lookup.templar_variables, [{"applications": {}}])
         self.assertIs(templar.available_variables, task_vars)
         self.assertEqual(task_vars["application_id"], "web-app-bluesky")
 
     def test_revalidates_the_tokens_file_before_rendering(self):
         order = []
-        lookup = _FakeLookup({"key": ["roles", []], "applications": {}})
-        lookup.run = lambda *args, **kwargs: order.append("render") or [{}]
+        applications, users, loader = _lookups()
+        applications.run = lambda *args, **kwargs: (
+            order.append("applications") or [applications.carrier]
+        )
+        users.run = lambda *args, **kwargs: order.append("users") or [users.carrier]
         action = _make_action(_FakeTask())
         with (
             patch(
@@ -103,7 +181,7 @@ class TestApplicationsCarrierActionPlugin(unittest.TestCase):
             ),
             patch(
                 "plugins.action.applications_carrier.lookup_loader.get",
-                return_value=lookup,
+                side_effect=loader.get,
             ),
             patch(
                 "plugins.action.applications_carrier.revalidate_tokens_file",
@@ -113,7 +191,8 @@ class TestApplicationsCarrierActionPlugin(unittest.TestCase):
             action.run(task_vars={"FILE_TOKENS": "/x/tokens.yml"})
 
         self.assertEqual(
-            order, [("revalidate", {"FILE_TOKENS": "/x/tokens.yml"}), "render"]
+            order,
+            [("revalidate", {"FILE_TOKENS": "/x/tokens.yml"}), "applications", "users"],
         )
 
     def test_clear_does_not_revalidate(self):
@@ -144,8 +223,8 @@ class TestApplicationsCarrierActionPlugin(unittest.TestCase):
         ):
             action.run(task_vars={})
 
-    def test_clear_drops_the_fact_without_rendering(self):
-        lookup = _FakeLookup({"key": ["roles", []], "applications": {}})
+    def test_clear_drops_both_facts_without_rendering(self):
+        applications, users, loader = _lookups()
         action = _make_action(_FakeTask(args={"clear": True}))
         with (
             patch(
@@ -155,18 +234,21 @@ class TestApplicationsCarrierActionPlugin(unittest.TestCase):
             ),
             patch(
                 "plugins.action.applications_carrier.lookup_loader.get",
-                return_value=lookup,
+                side_effect=loader.get,
             ),
         ):
             result = action.run(task_vars={"applications": {}})
 
-        self.assertIsNone(result["ansible_facts"][APPLICATIONS_RENDERED_FACT])
+        self.assertEqual(
+            result["ansible_facts"],
+            {APPLICATIONS_RENDERED_FACT: None, USERS_RENDERED_FACT: None},
+        )
         self.assertFalse(result["changed"])
-        self.assertEqual(lookup.calls, [])
+        self.assertEqual(applications.calls, [])
+        self.assertEqual(users.calls, [])
 
     def test_clear_false_still_renders(self):
-        carrier = {"key": ["roles", ["a", "b", "c", "d"]], "applications": {"x": {}}}
-        lookup = _FakeLookup(carrier)
+        applications, users, loader = _lookups()
         action = _make_action(_FakeTask(args={"clear": "no"}))
         with (
             patch(
@@ -176,13 +258,17 @@ class TestApplicationsCarrierActionPlugin(unittest.TestCase):
             ),
             patch(
                 "plugins.action.applications_carrier.lookup_loader.get",
-                return_value=lookup,
+                side_effect=loader.get,
             ),
         ):
             result = action.run(task_vars={"applications": {}})
 
-        self.assertIs(result["ansible_facts"][APPLICATIONS_RENDERED_FACT], carrier)
-        self.assertEqual(len(lookup.calls), 1)
+        self.assertIs(
+            result["ansible_facts"][APPLICATIONS_RENDERED_FACT], applications.carrier
+        )
+        self.assertIs(result["ansible_facts"][USERS_RENDERED_FACT], users.carrier)
+        self.assertEqual(len(applications.calls), 1)
+        self.assertEqual(len(users.calls), 1)
 
 
 if __name__ == "__main__":

tests/unit/python/plugins/lookup/test_users.py

@@ -10,6 +10,7 @@ from ansible.errors import AnsibleError
 
 from plugins.lookup.users import LookupModule, _reset_cache_for_tests
 from utils.cache import base as runtime_data_base
+from utils.cache.carrier import USERS_RENDERED_FACT, merged_users_cache_key
 from utils.cache.yaml import dump_yaml_str
 from utils.paths import FILE_TOKENS
 from utils.roles.mapping import ROLE_FILE_META_USERS
@@ -303,6 +304,46 @@ class TestUsersLookup(unittest.TestCase):
 
         self.assertEqual(result["tokens"]["web-app-mailu"], "token-from-default-path")
 
+    def test_carrier_returns_the_users_with_the_key_they_are_valid_under(
+        self,
+    ) -> None:
+        roles = str(self._tmp / "roles")
+        variables = {"inventory_hostname": "h1"}
+        result = self.lookup.run(
+            [], variables=variables, roles_dir=roles, carrier=True
+        )[0]
+        key = merged_users_cache_key(variables, roles_dir=roles)
+        self.assertEqual(result["key"], [key[0], list(key[1])])
+        self.assertEqual(
+            result["users"],
+            self.lookup.run([], variables=variables, roles_dir=roles)[0],
+        )
+
+    def test_carrier_takes_no_terms(self) -> None:
+        with self.assertRaises(AnsibleError):
+            self.lookup.run(
+                ["alice"],
+                variables={},
+                roles_dir=str(self._tmp / "roles"),
+                carrier=True,
+            )
+
+    def test_a_parked_carrier_answers_a_fresh_worker_without_rendering(self) -> None:
+        roles = str(self._tmp / "roles")
+        self.lookup._templar = _DummyTemplar({"SYSTEM_EMAIL_DOMAIN": "example.org"})
+        variables = {"inventory_hostname": "h1", "SYSTEM_EMAIL_DOMAIN": "example.org"}
+        parked = self.lookup.run(
+            [], variables=variables, roles_dir=roles, carrier=True
+        )[0]
+        _reset_cache_for_tests()
+        variables[USERS_RENDERED_FACT] = parked
+        with patch(
+            "utils.cache.users._render_with_templar",
+            side_effect=AssertionError("rendered"),
+        ):
+            result = self.lookup.run(["alice"], variables=variables, roles_dir=roles)[0]
+        self.assertIs(result, parked["users"]["alice"])
+
 
 class _DummyTemplar:
     def __init__(self, variables: dict[str, str]) -> None: