mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-10 12:47:04 +00:00
feat(plugins): size memory parameters from the container limit with one lookup
lookup('service_memory_mb', application_id, service_name, share) returns
a share of a service's mem_limit in whole MiB. It resolves the limit the
way sys-svc-container resolves the container's own: the named service,
then the role's primary entity, then RESOURCE_MEM_LIMIT.
It replaces the three filters of memory_filters, each with its own unit
parser and its own idea of the limit: Redis maxmemory at 0.8 of the
limit, and the Confluence and Jira heap at 0.7 and 0.35.
The central Redis (409 MiB), Confluence (5734/2867) and Jira (4300/2150)
keep their values. A Redis sidecar without a mem_limit of its own does
not: the filter fell back to a fixed 256m, which gave 204 MiB, while the
lookup takes the limit that container really runs with, the primary
entity's. That moves 25 roles to between 409 and 3276 MiB.
The heap clamps of the old filters are gone: the 1024 MiB floor, limit
minus 1024, the 12288 cap and Xms bounded by mem_reservation.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
kevinveenbirkenbachandClaude Opus 5.5 committed
1 parent
09894d5d79
commit
e203d6f7ab
8 files changed
+213
-424
No files matched your search
@@ -1,234 +0,0 @@
|
||||
import importlib
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
memory_filters = importlib.import_module("plugins.filter.memory_filters")
|
||||
|
||||
|
||||
class TestMemoryFilters(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.apps = {"whatever": True}
|
||||
self.app_id = "web-app-confluence" # entity_name will be mocked
|
||||
|
||||
# -----------------------------
|
||||
# Helpers
|
||||
# -----------------------------
|
||||
def _with_conf(self, mem_limit: str, mem_res: str):
|
||||
"""
|
||||
Patch get/get_entity_name so that mem_limit and mem_reservation
|
||||
can be controlled in tests.
|
||||
"""
|
||||
patches = [
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get_entity_name",
|
||||
return_value="confluence",
|
||||
),
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get",
|
||||
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
|
||||
mem_limit
|
||||
if key.endswith(".mem_limit")
|
||||
else mem_res
|
||||
if key.endswith(".mem_reservation")
|
||||
else None
|
||||
),
|
||||
),
|
||||
]
|
||||
mocks = [p.start() for p in patches]
|
||||
self.addCleanup(lambda: [p.stop() for p in patches])
|
||||
return mocks
|
||||
|
||||
# -----------------------------
|
||||
# Tests: jvm_max_mb / jvm_min_mb sizing
|
||||
# -----------------------------
|
||||
def test_sizing_8g_limit_6g_reservation(self):
|
||||
self._with_conf("8g", "6g")
|
||||
xmx = memory_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
xms = memory_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
self.assertEqual(xmx, 5734)
|
||||
self.assertEqual(xms, 2867)
|
||||
|
||||
def test_sizing_6g_limit_4g_reservation(self):
|
||||
self._with_conf("6g", "4g")
|
||||
xmx = memory_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
xms = memory_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
self.assertEqual(xmx, 4300)
|
||||
self.assertEqual(xms, 2150)
|
||||
|
||||
def test_sizing_16g_limit_12g_reservation_cap_12288(self):
|
||||
self._with_conf("16g", "12g")
|
||||
xmx = memory_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
xms = memory_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
self.assertEqual(xmx, 11468)
|
||||
self.assertEqual(xms, 5734)
|
||||
|
||||
def test_floor_small_limit_results_in_min_1024(self):
|
||||
self._with_conf("1g", "512m")
|
||||
xmx = memory_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
self.assertEqual(xmx, 1024)
|
||||
|
||||
def test_floor_small_reservation_results_in_min_512(self):
|
||||
self._with_conf("4g", "128m")
|
||||
xms = memory_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
self.assertEqual(xms, 512)
|
||||
|
||||
# -----------------------------
|
||||
# Tests: JVM failure cases / validation
|
||||
# -----------------------------
|
||||
def test_invalid_unit_raises(self):
|
||||
with (
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get_entity_name",
|
||||
return_value="confluence",
|
||||
),
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get",
|
||||
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
|
||||
"8Q" if key.endswith(".mem_limit") else "4g"
|
||||
),
|
||||
),
|
||||
self.assertRaises(memory_filters.AnsibleFilterError),
|
||||
):
|
||||
memory_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
|
||||
def test_zero_limit_raises(self):
|
||||
with (
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get_entity_name",
|
||||
return_value="confluence",
|
||||
),
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get",
|
||||
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
|
||||
"0" if key.endswith(".mem_limit") else "4g"
|
||||
),
|
||||
),
|
||||
self.assertRaises(memory_filters.AnsibleFilterError),
|
||||
):
|
||||
memory_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
|
||||
def test_zero_reservation_raises(self):
|
||||
with (
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get_entity_name",
|
||||
return_value="confluence",
|
||||
),
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get",
|
||||
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
|
||||
"8g" if key.endswith(".mem_limit") else "0"
|
||||
),
|
||||
),
|
||||
self.assertRaises(memory_filters.AnsibleFilterError),
|
||||
):
|
||||
memory_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
|
||||
def test_entity_name_is_derived_not_passed(self):
|
||||
"""
|
||||
Ensure get_entity_name() is called internally and the app_id is not
|
||||
passed around manually from the template.
|
||||
"""
|
||||
with (
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get_entity_name",
|
||||
return_value="confluence",
|
||||
) as mock_entity,
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get",
|
||||
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
|
||||
"8g" if key.endswith(".mem_limit") else "6g"
|
||||
),
|
||||
),
|
||||
):
|
||||
xmx = memory_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
xms = memory_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
self.assertGreater(xmx, 0)
|
||||
self.assertGreater(xms, 0)
|
||||
self.assertEqual(mock_entity.call_count, 3)
|
||||
for call in mock_entity.call_args_list:
|
||||
self.assertEqual(call.args[0], self.app_id)
|
||||
|
||||
# -----------------------------
|
||||
# Tests: redis_maxmemory_mb
|
||||
# -----------------------------
|
||||
def test_redis_maxmemory_default_factor_uses_80_percent_of_limit(self):
|
||||
self._with_conf("1g", "512m")
|
||||
maxmem = memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
|
||||
self.assertEqual(maxmem, 819)
|
||||
|
||||
def test_redis_maxmemory_custom_factor_and_min_mb(self):
|
||||
self._with_conf("1g", "512m")
|
||||
maxmem = memory_filters.redis_maxmemory_mb(
|
||||
self.apps,
|
||||
self.app_id,
|
||||
factor=0.5,
|
||||
min_mb=128,
|
||||
)
|
||||
self.assertEqual(maxmem, 512)
|
||||
|
||||
def test_redis_maxmemory_honors_minimum_floor(self):
|
||||
self._with_conf("32m", "16m")
|
||||
maxmem = memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
|
||||
self.assertEqual(maxmem, 64)
|
||||
|
||||
def test_redis_maxmemory_zero_limit_raises(self):
|
||||
self._with_conf("0", "512m")
|
||||
with self.assertRaises(memory_filters.AnsibleFilterError):
|
||||
memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
|
||||
|
||||
def test_redis_maxmemory_invalid_unit_raises(self):
|
||||
self._with_conf("8Q", "512m")
|
||||
with self.assertRaises(memory_filters.AnsibleFilterError):
|
||||
memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
|
||||
|
||||
def test_redis_maxmemory_does_not_call_get_entity_name(self):
|
||||
"""
|
||||
Ensure redis_maxmemory_mb does NOT rely on entity name resolution
|
||||
(it should always use the hard-coded 'redis' service name).
|
||||
"""
|
||||
patches = [
|
||||
patch("plugins.filter.memory_filters.get_entity_name"),
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get",
|
||||
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
|
||||
"4g" if key.endswith(".mem_limit") else "2g"
|
||||
),
|
||||
),
|
||||
]
|
||||
mocks = [p.start() for p in patches]
|
||||
self.addCleanup(lambda: [p.stop() for p in patches])
|
||||
|
||||
entity_mock = mocks[0]
|
||||
|
||||
maxmem = memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
|
||||
self.assertEqual(maxmem, 3276)
|
||||
entity_mock.assert_not_called()
|
||||
|
||||
def test_redis_maxmemory_uses_default_when_mem_limit_missing(self):
|
||||
"""
|
||||
When services.redis.mem_limit is not configured, the filter
|
||||
should fall back to its internal default (256m).
|
||||
"""
|
||||
|
||||
def fake_get(apps, app_id, key, required=True, **kwargs):
|
||||
if key.endswith(".mem_limit"):
|
||||
return kwargs.get("default")
|
||||
return None
|
||||
|
||||
with (
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get",
|
||||
side_effect=fake_get,
|
||||
),
|
||||
patch(
|
||||
"plugins.filter.memory_filters.get_entity_name",
|
||||
return_value="confluence",
|
||||
),
|
||||
):
|
||||
maxmem = memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
|
||||
|
||||
self.assertEqual(maxmem, 204)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from ansible.errors import AnsibleError
|
||||
|
||||
from plugins.lookup.service_memory_mb import LookupModule, docker_bytes, share_mb
|
||||
|
||||
_APP = "web-app-demo"
|
||||
|
||||
|
||||
class TestDockerBytes(unittest.TestCase):
|
||||
def test_suffixes_are_binary(self) -> None:
|
||||
self.assertEqual(docker_bytes("512m"), 512 * 1024**2)
|
||||
self.assertEqual(docker_bytes("2g"), 2 * 1024**3)
|
||||
self.assertEqual(docker_bytes("1.5g"), 1536 * 1024**2)
|
||||
self.assertEqual(docker_bytes("64K"), 64 * 1024)
|
||||
self.assertEqual(docker_bytes("1gb"), 1024**3)
|
||||
|
||||
def test_a_bare_number_is_bytes(self) -> None:
|
||||
self.assertEqual(docker_bytes(1048576), 1024**2)
|
||||
self.assertEqual(docker_bytes("2048"), 2048)
|
||||
|
||||
def test_an_unreadable_size_raises(self) -> None:
|
||||
for size in ("", "lots", "2x", "-1g", None):
|
||||
with self.subTest(size=size), self.assertRaises(AnsibleError):
|
||||
docker_bytes(size)
|
||||
|
||||
|
||||
class TestShareMb(unittest.TestCase):
|
||||
def test_share_of_the_limit_in_whole_mebibytes(self) -> None:
|
||||
cases = (
|
||||
("2g", 0.5, 1024),
|
||||
("4g", 0.5, 2048),
|
||||
("8g", 0.7, 5734),
|
||||
("8g", 0.35, 2867),
|
||||
("6g", 0.7, 4300),
|
||||
("512m", 0.8, 409),
|
||||
("256m", 0.75, 192),
|
||||
("1.5g", 0.5, 768),
|
||||
("1g", 1, 1024),
|
||||
)
|
||||
for mem_limit, share, expected in cases:
|
||||
with self.subTest(mem_limit=mem_limit, share=share):
|
||||
self.assertEqual(share_mb(mem_limit, share), expected)
|
||||
|
||||
def test_a_share_outside_the_limit_raises(self) -> None:
|
||||
for share in (0, -0.5, 1.01, "half", None):
|
||||
with self.subTest(share=share), self.assertRaises(AnsibleError):
|
||||
share_mb("2g", share)
|
||||
|
||||
def test_a_result_below_one_mebibyte_raises(self) -> None:
|
||||
with self.assertRaises(AnsibleError):
|
||||
share_mb("1m", 0.5)
|
||||
|
||||
|
||||
class TestLookupResolvesTheContainerLimit(unittest.TestCase):
|
||||
def _run(self, services: dict, variables: dict, terms: list) -> int:
|
||||
templar = MagicMock()
|
||||
templar.available_variables = variables
|
||||
templar.template.side_effect = lambda value: {"{{ HOST }}": "1.5g"}.get(
|
||||
value, value
|
||||
)
|
||||
applications = MagicMock()
|
||||
applications.run.return_value = [{_APP: {"services": services}}]
|
||||
lookup = LookupModule(loader=None, templar=templar)
|
||||
with patch(
|
||||
"plugins.lookup.service_memory_mb.lookup_loader.get",
|
||||
return_value=applications,
|
||||
):
|
||||
return lookup.run(terms, variables=variables)[0]
|
||||
|
||||
def test_the_named_service_wins(self) -> None:
|
||||
services = {"search": {"mem_limit": "2g"}, "demo": {"mem_limit": "8g"}}
|
||||
|
||||
self.assertEqual(self._run(services, {}, [_APP, "search", 0.5]), 1024)
|
||||
|
||||
def test_the_primary_entity_backs_an_undeclared_service(self) -> None:
|
||||
services = {"search": {}, "demo": {"mem_limit": "8g"}}
|
||||
|
||||
self.assertEqual(self._run(services, {}, [_APP, "search", 0.5]), 4096)
|
||||
|
||||
def test_the_host_default_backs_an_undeclared_role(self) -> None:
|
||||
variables = {"RESOURCE_MEM_LIMIT": "{{ HOST }}"}
|
||||
|
||||
self.assertEqual(
|
||||
self._run({"search": {}}, variables, [_APP, "search", 0.5]), 768
|
||||
)
|
||||
|
||||
def test_no_limit_anywhere_raises(self) -> None:
|
||||
with self.assertRaises(AnsibleError):
|
||||
self._run({"search": {}}, {}, [_APP, "search", 0.5])
|
||||
|
||||
def test_the_term_count_is_checked(self) -> None:
|
||||
with self.assertRaises(AnsibleError):
|
||||
self._run({"search": {"mem_limit": "2g"}}, {}, [_APP, "search"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user