Files
computer-playbook/tests/unit/python/utils/cache/test_base.py
kevinveenbirkenbachandClaude Opus 5.5 67f6c8d7e8 fix(cache): keep the applications carrier valid across token store writes
Since 778c18fb23 the carrier key holds the tokens file's mtime and size.
sys-token-store changed that stat on every inclusion (00_core.yml
touched the file) and on every persisted token, and nothing re-parked a
carrier afterwards. Every later task's fresh worker then rendered the
applications SPOT cold, 30 to 80 s each. On main fea6f31d9f (run
38812964308) seven heavy jobs logged 194 to 287 such tasks and ran out
of the 20400 s budget; the same jobs before 778c18fb23 logged 65 to 83,
all of them real waits.

- 00_core.yml touches the store with modification_time and access_time
  preserved, so it only creates a missing file. In swarm the controller
  sees the store over NFS with attribute caching, and the touch alone
  made the second pass go cold without any write.
- write.yml re-parks the carrier when the persist or the controller
  mirror changed the store. Both run once, so every host that ran the
  include gets the same verdict.
- tasks/utils/setup/role.yml re-parks after every role on every host,
  so hosts that did not run the writing role recover too. Where the
  key still matches this serves the carried payload; measured 0.1 s per
  role in a local compose deploy.
- The tokens file enters the key as the SHA-256 of its content,
  memoised per inode, mtime and size, so a touch no longer changes the
  key. A stat that still matches serves the memo (1.67 us against
  1.41 us for the old stat signature), and a stale stat keeps key and
  YAML cache on the same old content instead of pairing new content
  with an old key.
- applications_carrier opens the tokens file before it computes the
  key, so on the NFS-mounted controller the parked key reflects the
  revalidated attributes rather than a cached stat that changes a
  minute later.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-16 11:11:47 +02:00

251 lines
9.3 KiB
Python

"""Focused unit tests for ``utils.cache.base``.
Covers the cross-cutting helpers in isolation: constants, deep_merge,
yaml-loaders, signatures, content fingerprints, _reset(), and the
templar-render machinery (with both no-templar and stub-templar
inputs). The broader integration of these helpers via the
applications/users/domains paths is covered in the per-domain test
files; this file pins the contract of base itself.
"""
from __future__ import annotations
import os
import tempfile
import textwrap
import unittest
import unittest.mock as mock
from pathlib import Path
from utils.cache import _reset_cache_for_tests, base
from utils.paths import FILE_TOKENS
def _write(path: Path, content: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(textwrap.dedent(content), encoding="utf-8")
class TestProjectRootInvariants(unittest.TestCase):
"""`base.PROJECT_ROOT` must point at the actual repo root, not at
`<repo>/utils/`. CI run 24934007615 was caused by exactly this
drift after `utils/cache/data.py` was moved one level deeper.
"""
def test_project_root_resolves_to_repo_root(self):
self.assertTrue((base.PROJECT_ROOT / "roles").is_dir())
self.assertTrue((base.PROJECT_ROOT / "cli").is_dir())
self.assertEqual(base.ROLES_DIR, base.PROJECT_ROOT / "roles")
def test_default_tokens_file_is_secrets_yaml(self):
self.assertEqual(base.DEFAULT_TOKENS_FILE, FILE_TOKENS)
class TestDeepMerge(unittest.TestCase):
def test_overrides_win_on_scalar_keys(self):
result = base._deep_merge({"a": 1, "b": 2}, {"b": 99})
self.assertEqual(result, {"a": 1, "b": 99})
def test_recurses_into_nested_mappings(self):
result = base._deep_merge(
{"x": {"y": 1, "z": 2}},
{"x": {"y": 99}},
)
self.assertEqual(result, {"x": {"y": 99, "z": 2}})
def test_override_replaces_when_types_differ(self):
result = base._deep_merge({"x": {"y": 1}}, {"x": ["a", "b"]})
self.assertEqual(result, {"x": ["a", "b"]})
def test_returns_deep_copy_of_override_when_base_is_none(self):
override = {"x": [1, 2, 3]}
result = base._deep_merge(None, override)
self.assertEqual(result, override)
result["x"].append(4)
self.assertEqual(override["x"], [1, 2, 3])
class TestResolveRolesDir(unittest.TestCase):
def test_explicit_arg_wins(self):
with tempfile.TemporaryDirectory() as tmp:
self.assertEqual(
base._resolve_roles_dir(roles_dir=tmp), Path(tmp).resolve()
)
def test_falls_back_to_module_default(self):
self.assertEqual(base._resolve_roles_dir(), base.ROLES_DIR.resolve())
class TestCacheKey(unittest.TestCase):
def test_returns_resolved_string(self):
with tempfile.TemporaryDirectory() as tmp:
self.assertEqual(base._cache_key(Path(tmp)), str(Path(tmp).resolve()))
class TestFingerprintMapping(unittest.TestCase):
def setUp(self) -> None:
_reset_cache_for_tests()
def test_none_short_circuits_to_zero(self):
self.assertEqual(base._fingerprint_mapping(None), "0")
def test_same_object_hashes_stably(self):
obj = {"a": 1}
first = base._fingerprint_mapping(obj)
self.assertEqual(first, base._fingerprint_mapping(obj))
def test_in_place_mutation_changes_the_digest(self):
obj = {"a": 1}
first = base._fingerprint_mapping(obj)
obj["a"] = 2
self.assertNotEqual(first, base._fingerprint_mapping(obj))
def test_equal_content_distinct_objects_hash_to_same_digest(self):
a = {"a": 1, "b": 2}
b = {"b": 2, "a": 1}
self.assertEqual(base._fingerprint_mapping(a), base._fingerprint_mapping(b))
class TestStableVariablesSignature(unittest.TestCase):
def setUp(self) -> None:
_reset_cache_for_tests()
def test_empty_variables_collapses_to_canonical_tuple(self):
self.assertEqual(
base._stable_variables_signature(None),
("0", "0", "", "", ""),
)
self.assertEqual(
base._stable_variables_signature({}),
("0", "0", "", "", ""),
)
def test_two_hosts_of_one_play_do_not_share_a_key(self):
shared = {"applications": {"web-app-pretix": {}}, "DOMAIN_PRIMARY": "x.test"}
member = base._stable_variables_signature(
{**shared, "group_names": ["web-app-pretix", "svc-db-postgres"]}
)
outsider = base._stable_variables_signature(
{**shared, "group_names": ["web-app-pretix"]}
)
self.assertNotEqual(
member,
outsider,
"a service flag may read group_names, so one host's payload must "
"never answer for a host whose groups differ",
)
def test_group_order_does_not_split_the_key(self):
first = base._stable_variables_signature({"group_names": ["a", "b"]})
second = base._stable_variables_signature({"group_names": ["b", "a"]})
self.assertEqual(first, second)
def test_includes_domain_primary_and_email_domain_strings(self):
sig = base._stable_variables_signature(
{
"applications": {"web-app-x": {}},
"DOMAIN_PRIMARY": "infinito.test",
"SYSTEM_EMAIL_DOMAIN": "mail.infinito.test",
}
)
self.assertEqual(sig[2], "infinito.test")
self.assertEqual(sig[3], "mail.infinito.test")
class TestTokensFileSignature(unittest.TestCase):
def setUp(self):
base._reset()
def test_missing_file_returns_an_empty_digest(self):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "absent.yml"
self.assertEqual(base._tokens_file_signature(path), (str(path), ""))
def test_signature_changes_when_file_changes(self):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "tokens.yml"
path.write_text("a: 1\n", encoding="utf-8")
sig_before = base._tokens_file_signature(path)
path.write_text("a: 1\nb: 2\n", encoding="utf-8")
sig_after = base._tokens_file_signature(path)
self.assertNotEqual(sig_before, sig_after)
def test_a_touch_keeps_the_signature(self):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "tokens.yml"
path.write_text("a: 1\n", encoding="utf-8")
sig_before = base._tokens_file_signature(path)
later = path.stat().st_mtime_ns + 5_000_000_000
os.utime(path, ns=(later, later))
self.assertEqual(base._tokens_file_signature(path), sig_before)
def test_an_unchanged_stat_serves_the_digest_without_reading(self):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "tokens.yml"
path.write_text("a: 1\n", encoding="utf-8")
sig_before = base._tokens_file_signature(path)
with mock.patch.object(Path, "open", side_effect=AssertionError):
self.assertEqual(base._tokens_file_signature(path), sig_before)
class TestRevalidateTokensFile(unittest.TestCase):
def test_opens_the_file(self):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "tokens.yml"
path.write_text("a: 1\n", encoding="utf-8")
with mock.patch.object(Path, "open", wraps=path.open) as opened:
base._revalidate_tokens_file(path)
opened.assert_called_once_with("rb")
def test_a_missing_file_is_ignored(self):
with tempfile.TemporaryDirectory() as tmp:
base._revalidate_tokens_file(Path(tmp) / "absent.yml")
class TestRenderWithTemplar(unittest.TestCase):
"""`_render_with_templar` is the single ansible-coupled symbol in
base. We don't simulate a real Templar (that's covered by the
domain integration tests); we pin the no-templar short-circuit and
the basic stub-templar dispatch shape so a regression in the
closure setup or available_variables push/pop trips here.
"""
def test_returns_value_unchanged_when_templar_is_none(self):
sentinel = {"a": 1, "b": [2, 3]}
result = base._render_with_templar(sentinel, templar=None, variables={"x": 1})
self.assertIs(result, sentinel)
class TestResolveOverrideMapping(unittest.TestCase):
def test_missing_key_returns_empty_dict(self):
self.assertEqual(base._resolve_override_mapping({}, "applications"), {})
def test_mapping_passes_through(self):
result = base._resolve_override_mapping(
{"applications": {"web-app-x": {"foo": 1}}}, "applications"
)
self.assertEqual(result, {"web-app-x": {"foo": 1}})
def test_falls_back_to_raw_inventory_when_value_is_not_mapping(self):
result = base._resolve_override_mapping(
{
"applications": "<placeholder>",
"_INFINITO_APPLICATIONS_RAW": {"web-app-x": {"foo": 1}},
},
"applications",
)
self.assertEqual(result, {"web-app-x": {"foo": 1}})
class TestFingerprintIsContentOnly(unittest.TestCase):
def test_fingerprint_survives_reset_because_it_holds_no_state(self):
before = base._fingerprint_mapping({"a": 1})
_reset_cache_for_tests()
self.assertEqual(before, base._fingerprint_mapping({"a": 1}))
if __name__ == "__main__":
unittest.main()