From 7d9cb5820f15505efe947c0fea190098ec0e0a17 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 24 Sep 2025 11:29:40 +0200 Subject: [PATCH] feat(jvm): add robust JVM sizing filters and apply across Confluence/Jira Introduce filter_plugins/jvm_filters.py with jvm_max_mb/jvm_min_mb. Derive Xmx/Xms from docker mem_limit/mem_reservation using safe rules: Xmx=min(70% limit, limit-1024MB, 12288MB), floored at 1024MB; Xms=min(Xmx/2, reservation, Xmx), floored at 512MB. Parse human-readable sizes (k/m/g/t) with binary units. Wire filters into roles: set JVM_MINIMUM_MEMORY/JVM_MAXIMUM_MEMORY via filters; stop relying on host RAM. Keep env templates simple and stable. Add unit tests under tests/unit/filter_plugins/test_jvm_filters.py covering typical sizes, floors, caps, invalid inputs, and entity-name derivation. Ref: https://chatgpt.com/share/68d3b9f6-8d18-800f-aa8d-8a743ddf164d --- filter_plugins/jvm_filters.py | 77 +++++++++++ roles/web-app-confluence/vars/main.yml | 17 +-- roles/web-app-jira/vars/main.yml | 16 +-- tests/unit/filter_plugins/test_jvm_filters.py | 123 ++++++++++++++++++ 4 files changed, 214 insertions(+), 19 deletions(-) create mode 100644 filter_plugins/jvm_filters.py create mode 100644 tests/unit/filter_plugins/test_jvm_filters.py diff --git a/filter_plugins/jvm_filters.py b/filter_plugins/jvm_filters.py new file mode 100644 index 00000000..51fd1af9 --- /dev/null +++ b/filter_plugins/jvm_filters.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import sys, os, re +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from ansible.errors import AnsibleFilterError +from module_utils.config_utils import get_app_conf +from module_utils.entity_name_utils import get_entity_name + +_UNIT_RE = re.compile(r'^\s*(\d+(?:\.\d+)?)\s*([kKmMgGtT]?[bB]?)?\s*$') +_FACTORS = { + '': 1, 'b': 1, + 'k': 1024, 'kb': 1024, + 'm': 1024**2, 'mb': 1024**2, + 'g': 1024**3, 'gb': 1024**3, + 't': 1024**4, 'tb': 1024**4, +} + +def _to_bytes(v: str) -> int: + if v is None: + raise AnsibleFilterError("jvm_filters: size value is None") + s = str(v).strip() + m = _UNIT_RE.match(s) + if not m: + raise AnsibleFilterError(f"jvm_filters: invalid size '{v}'") + num, unit = m.group(1), (m.group(2) or '').lower() + try: + val = float(num) + except ValueError as e: + raise AnsibleFilterError(f"jvm_filters: invalid numeric size '{v}'") from e + factor = _FACTORS.get(unit) + if factor is None: + raise AnsibleFilterError(f"jvm_filters: unknown unit in '{v}'") + return int(val * factor) + +def _to_mb(v: str) -> int: + return max(0, _to_bytes(v) // (1024 * 1024)) + +def _svc(app_id: str) -> str: + return get_entity_name(app_id) + +def _mem_limit_mb(apps: dict, app_id: str) -> int: + svc = _svc(app_id) + raw = get_app_conf(apps, app_id, f"docker.services.{svc}.mem_limit") + mb = _to_mb(raw) + if mb <= 0: + raise AnsibleFilterError(f"jvm_filters: mem_limit for '{svc}' must be > 0 MB (got '{raw}')") + return mb + +def _mem_res_mb(apps: dict, app_id: str) -> int: + svc = _svc(app_id) + raw = get_app_conf(apps, app_id, f"docker.services.{svc}.mem_reservation") + mb = _to_mb(raw) + if mb <= 0: + raise AnsibleFilterError(f"jvm_filters: mem_reservation for '{svc}' must be > 0 MB (got '{raw}')") + return mb + +def jvm_max_mb(apps: dict, app_id: str) -> int: + """Xmx = min( floor(0.7*limit), limit-1024, 12288 ) with floor at 1024 MB.""" + limit_mb = _mem_limit_mb(apps, app_id) + c1 = (limit_mb * 7) // 10 + c2 = max(0, limit_mb - 1024) + c3 = 12288 + return max(1024, min(c1, c2, c3)) + +def jvm_min_mb(apps: dict, app_id: str) -> int: + """Xms = min( floor(Xmx/2), mem_reservation, Xmx ) with floor at 512 MB.""" + xmx = jvm_max_mb(apps, app_id) + res = _mem_res_mb(apps, app_id) + return max(512, min(xmx // 2, res, xmx)) + +class FilterModule(object): + def filters(self): + return { + "jvm_max_mb": jvm_max_mb, + "jvm_min_mb": jvm_min_mb, + } diff --git a/roles/web-app-confluence/vars/main.yml b/roles/web-app-confluence/vars/main.yml index 0431c72b..d5341f51 100644 --- a/roles/web-app-confluence/vars/main.yml +++ b/roles/web-app-confluence/vars/main.yml @@ -1,6 +1,7 @@ # General application_id: "web-app-confluence" database_type: "postgres" +entity_name: "{{ application_id | get_entity_name }}" # Container container_port: 8090 @@ -28,19 +29,15 @@ CONFLUENCE_OIDC_SCOPES: "openid,email,profile" CONFLUENCE_OIDC_UNIQUE_ATTRIBUTE: "{{ OIDC.ATTRIBUTES.USERNAME }}" ## Docker -CONFLUENCE_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.confluence.version') }}" -CONFLUENCE_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.confluence.image') }}" -CONFLUENCE_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.confluence.name') }}" +CONFLUENCE_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.version') }}" +CONFLUENCE_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.image') }}" +CONFLUENCE_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.name') }}" CONFLUENCE_DATA_VOLUME: "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}" CONFLUENCE_CUSTOM_IMAGE: "{{ CONFLUENCE_IMAGE }}_custom" -## Performance -CONFLUENCE_TOTAL_MB: "{{ ansible_memtotal_mb | int }}" -CONFLUENCE_JVM_MAX_MB: "{{ [ (CONFLUENCE_TOTAL_MB | int // 2), 12288 ] | min }}" -CONFLUENCE_JVM_MIN_MB: "{{ [ (CONFLUENCE_TOTAL_MB | int // 4), (CONFLUENCE_JVM_MAX_MB | int) ] | min }}" -CONFLUENCE_JVM_MIN: "{{ CONFLUENCE_JVM_MIN_MB }}m" -CONFLUENCE_JVM_MAX: "{{ CONFLUENCE_JVM_MAX_MB }}m" - +## Performance (derive from container limits in config/main.yml) +CONFLUENCE_JVM_MAX: "{{ applications | jvm_max_mb(application_id) }}m" +CONFLUENCE_JVM_MIN: "{{ applications | jvm_min_mb(application_id) }}m" ## Options CONFLUENCE_TRUST_STORE_ENABLED: "{{ applications | get_app_conf(application_id, 'truststore_enabled') }}" \ No newline at end of file diff --git a/roles/web-app-jira/vars/main.yml b/roles/web-app-jira/vars/main.yml index 0fde8eae..a60b3f93 100644 --- a/roles/web-app-jira/vars/main.yml +++ b/roles/web-app-jira/vars/main.yml @@ -1,6 +1,7 @@ # General application_id: "web-app-jira" database_type: "postgres" +entity_name: "{{ application_id | get_entity_name }}" # Container container_port: 8080 # Standardport Jira @@ -28,15 +29,12 @@ JIRA_OIDC_SCOPES: "openid,email,profile" JIRA_OIDC_UNIQUE_ATTRIBUTE: "{{ OIDC.ATTRIBUTES.USERNAME }}" ## Docker -JIRA_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.jira.version') }}" -JIRA_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.jira.image') }}" -JIRA_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.jira.name') }}" +JIRA_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.version') }}" +JIRA_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.image') }}" +JIRA_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.name') }}" JIRA_DATA_VOLUME: "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}" JIRA_CUSTOM_IMAGE: "{{ JIRA_IMAGE }}_custom" -## Performance (auto-derive from host memory) -JIRA_TOTAL_MB: "{{ ansible_memtotal_mb | int }}" -JIRA_JVM_MAX_MB: "{{ [ (JIRA_TOTAL_MB | int // 2), 12288 ] | min }}" -JIRA_JVM_MIN_MB: "{{ [ (JIRA_TOTAL_MB | int // 4), (JIRA_JVM_MAX_MB | int) ] | min }}" -JIRA_JVM_MIN: "{{ JIRA_JVM_MIN_MB }}m" -JIRA_JVM_MAX: "{{ JIRA_JVM_MAX_MB }}m" \ No newline at end of file +## Performance (derive from container limits in config/main.yml) +JIRA_JVM_MAX: "{{ applications | jvm_max_mb(application_id) }}m" +JIRA_JVM_MIN: "{{ applications | jvm_min_mb(application_id) }}m" \ No newline at end of file diff --git a/tests/unit/filter_plugins/test_jvm_filters.py b/tests/unit/filter_plugins/test_jvm_filters.py new file mode 100644 index 00000000..59bfad4b --- /dev/null +++ b/tests/unit/filter_plugins/test_jvm_filters.py @@ -0,0 +1,123 @@ +import unittest +from unittest.mock import patch + +# Importiere das Filtermodul +# Pfad relativ zum Projekt; falls nötig, passe den Importpfad an +import importlib +jvm_filters = importlib.import_module("filter_plugins.jvm_filters") + + +class TestJvmFilters(unittest.TestCase): + def setUp(self): + # Dummy applications dict – Inhalt egal, da get_app_conf gemockt wird + self.apps = {"whatever": True} + self.app_id = "web-app-confluence" # entity_name wird gemockt + + # ----------------------------- + # Helpers + # ----------------------------- + def _with_conf(self, mem_limit: str, mem_res: str): + """ + Context manager der get_app_conf/get_entity_name passend patched. + """ + patches = [ + patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"), + patch( + "filter_plugins.jvm_filters.get_app_conf", + side_effect=lambda apps, app_id, key, required=True: ( + mem_limit if key.endswith(".mem_limit") + else mem_res if key.endswith(".mem_reservation") + else None + ), + ), + ] + ctxs = [p.start() for p in patches] + self.addCleanup(lambda: [p.stop() for p in patches]) + return ctxs + + # ----------------------------- + # Tests: jvm_max_mb / jvm_min_mb Sizing + # ----------------------------- + def test_sizing_8g_limit_6g_reservation(self): + # mem_limit=8g → candidates: 70% = 5734MB (floor 8*0.7=5.6GB→ 5734MB via int math 8*7//10=5) + # int math: (8*1024)*7//10 = (8192)*7//10 = 5734 + # limit-1024 = 8192-1024 = 7168 + # 12288 + # → Xmx = min(5734, 7168, 12288) = 5734 → floor at 1024 keeps 5734 + # Xms = min(Xmx//2=2867, res=6144, Xmx=5734) = 2867 (>=512) + self._with_conf("8g", "6g") + xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id) + xms = jvm_filters.jvm_min_mb(self.apps, self.app_id) + self.assertEqual(xmx, 5734) + self.assertEqual(xms, 2867) + + def test_sizing_6g_limit_4g_reservation(self): + # limit=6g → 70%: (6144*7)//10 = 4300, limit-1024=5120, 12288 → Xmx=4300 + # Xms=min(4300//2=2150, 4096, 4300)=2150 + self._with_conf("6g", "4g") + xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id) + xms = jvm_filters.jvm_min_mb(self.apps, self.app_id) + self.assertEqual(xmx, 4300) + self.assertEqual(xms, 2150) + + def test_sizing_16g_limit_12g_reservation_cap_12288(self): + # limit=16g → 70%: (16384*7)//10 = 11468, limit-1024=15360, cap=12288 → Xmx=min(11468,15360,12288)=11468 + # Xms=min(11468//2=5734, 12288 (12g), 11468) = 5734 + self._with_conf("16g", "12g") + xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id) + xms = jvm_filters.jvm_min_mb(self.apps, self.app_id) + self.assertEqual(xmx, 11468) + self.assertEqual(xms, 5734) + + def test_floor_small_limit_results_in_min_1024(self): + # limit=1g → 70%: 716, limit-1024=0, 12288 → min=0 → floor → 1024 + self._with_conf("1g", "512m") + xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id) + self.assertEqual(xmx, 1024) + + def test_floor_small_reservation_results_in_min_512(self): + # limit groß genug, aber reservation sehr klein → Xms floored to 512 + self._with_conf("4g", "128m") + xms = jvm_filters.jvm_min_mb(self.apps, self.app_id) + self.assertEqual(xms, 512) + + # ----------------------------- + # Tests: Fehlerfälle / Validierung + # ----------------------------- + def test_invalid_unit_raises(self): + with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"), \ + patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True: + "8Q" if key.endswith(".mem_limit") else "4g"): + with self.assertRaises(jvm_filters.AnsibleFilterError): + jvm_filters.jvm_max_mb(self.apps, self.app_id) + + def test_zero_limit_raises(self): + with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"), \ + patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True: + "0" if key.endswith(".mem_limit") else "4g"): + with self.assertRaises(jvm_filters.AnsibleFilterError): + jvm_filters.jvm_max_mb(self.apps, self.app_id) + + def test_zero_reservation_raises(self): + with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"), \ + patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True: + "8g" if key.endswith(".mem_limit") else "0"): + with self.assertRaises(jvm_filters.AnsibleFilterError): + jvm_filters.jvm_min_mb(self.apps, self.app_id) + + def test_entity_name_is_derived_not_passed(self): + # Sicherstellen, dass get_entity_name() aufgerufen wird und kein externer Parameter nötig ist + with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence") as mock_entity, \ + patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True: + "8g" if key.endswith(".mem_limit") else "6g"): + xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id) + xms = jvm_filters.jvm_min_mb(self.apps, self.app_id) + self.assertGreater(xmx, 0) + self.assertGreater(xms, 0) + self.assertEqual(mock_entity.call_count, 3) + for call in mock_entity.call_args_list: + self.assertEqual(call.args[0], self.app_id) + + +if __name__ == "__main__": + unittest.main()