mirror of
				https://github.com/kevinveenbirkenbach/computer-playbook.git
				synced 2025-10-31 02:10:05 +00:00 
			
		
		
		
	feat(jvm): add robust JVM sizing filters and apply across Confluence/Jira
Introduce filter_plugins/jvm_filters.py with jvm_max_mb/jvm_min_mb. Derive Xmx/Xms from docker mem_limit/mem_reservation using safe rules: Xmx=min(70% limit, limit-1024MB, 12288MB), floored at 1024MB; Xms=min(Xmx/2, reservation, Xmx), floored at 512MB. Parse human-readable sizes (k/m/g/t) with binary units. Wire filters into roles: set JVM_MINIMUM_MEMORY/JVM_MAXIMUM_MEMORY via filters; stop relying on host RAM. Keep env templates simple and stable. Add unit tests under tests/unit/filter_plugins/test_jvm_filters.py covering typical sizes, floors, caps, invalid inputs, and entity-name derivation. Ref: https://chatgpt.com/share/68d3b9f6-8d18-800f-aa8d-8a743ddf164d
This commit is contained in:
		
							
								
								
									
										77
									
								
								filter_plugins/jvm_filters.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										77
									
								
								filter_plugins/jvm_filters.py
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,77 @@ | ||||
| from __future__ import annotations | ||||
|  | ||||
| import sys, os, re | ||||
| sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) | ||||
|  | ||||
| from ansible.errors import AnsibleFilterError | ||||
| from module_utils.config_utils import get_app_conf | ||||
| from module_utils.entity_name_utils import get_entity_name | ||||
|  | ||||
| _UNIT_RE = re.compile(r'^\s*(\d+(?:\.\d+)?)\s*([kKmMgGtT]?[bB]?)?\s*$') | ||||
| _FACTORS = { | ||||
|     '': 1, 'b': 1, | ||||
|     'k': 1024, 'kb': 1024, | ||||
|     'm': 1024**2, 'mb': 1024**2, | ||||
|     'g': 1024**3, 'gb': 1024**3, | ||||
|     't': 1024**4, 'tb': 1024**4, | ||||
| } | ||||
|  | ||||
| def _to_bytes(v: str) -> int: | ||||
|     if v is None: | ||||
|         raise AnsibleFilterError("jvm_filters: size value is None") | ||||
|     s = str(v).strip() | ||||
|     m = _UNIT_RE.match(s) | ||||
|     if not m: | ||||
|         raise AnsibleFilterError(f"jvm_filters: invalid size '{v}'") | ||||
|     num, unit = m.group(1), (m.group(2) or '').lower() | ||||
|     try: | ||||
|         val = float(num) | ||||
|     except ValueError as e: | ||||
|         raise AnsibleFilterError(f"jvm_filters: invalid numeric size '{v}'") from e | ||||
|     factor = _FACTORS.get(unit) | ||||
|     if factor is None: | ||||
|         raise AnsibleFilterError(f"jvm_filters: unknown unit in '{v}'") | ||||
|     return int(val * factor) | ||||
|  | ||||
| def _to_mb(v: str) -> int: | ||||
|     return max(0, _to_bytes(v) // (1024 * 1024)) | ||||
|  | ||||
| def _svc(app_id: str) -> str: | ||||
|     return get_entity_name(app_id) | ||||
|  | ||||
| def _mem_limit_mb(apps: dict, app_id: str) -> int: | ||||
|     svc = _svc(app_id) | ||||
|     raw = get_app_conf(apps, app_id, f"docker.services.{svc}.mem_limit") | ||||
|     mb = _to_mb(raw) | ||||
|     if mb <= 0: | ||||
|         raise AnsibleFilterError(f"jvm_filters: mem_limit for '{svc}' must be > 0 MB (got '{raw}')") | ||||
|     return mb | ||||
|  | ||||
| def _mem_res_mb(apps: dict, app_id: str) -> int: | ||||
|     svc = _svc(app_id) | ||||
|     raw = get_app_conf(apps, app_id, f"docker.services.{svc}.mem_reservation") | ||||
|     mb = _to_mb(raw) | ||||
|     if mb <= 0: | ||||
|         raise AnsibleFilterError(f"jvm_filters: mem_reservation for '{svc}' must be > 0 MB (got '{raw}')") | ||||
|     return mb | ||||
|  | ||||
| def jvm_max_mb(apps: dict, app_id: str) -> int: | ||||
|     """Xmx = min( floor(0.7*limit), limit-1024, 12288 ) with floor at 1024 MB.""" | ||||
|     limit_mb = _mem_limit_mb(apps, app_id) | ||||
|     c1 = (limit_mb * 7) // 10 | ||||
|     c2 = max(0, limit_mb - 1024) | ||||
|     c3 = 12288 | ||||
|     return max(1024, min(c1, c2, c3)) | ||||
|  | ||||
| def jvm_min_mb(apps: dict, app_id: str) -> int: | ||||
|     """Xms = min( floor(Xmx/2), mem_reservation, Xmx ) with floor at 512 MB.""" | ||||
|     xmx = jvm_max_mb(apps, app_id) | ||||
|     res = _mem_res_mb(apps, app_id) | ||||
|     return max(512, min(xmx // 2, res, xmx)) | ||||
|  | ||||
| class FilterModule(object): | ||||
|     def filters(self): | ||||
|         return { | ||||
|             "jvm_max_mb": jvm_max_mb, | ||||
|             "jvm_min_mb": jvm_min_mb, | ||||
|         } | ||||
| @@ -1,6 +1,7 @@ | ||||
| # General | ||||
| application_id:                   "web-app-confluence" | ||||
| database_type:                    "postgres" | ||||
| entity_name:                      "{{ application_id | get_entity_name }}" | ||||
|  | ||||
| # Container | ||||
| container_port:                   8090 | ||||
| @@ -28,19 +29,15 @@ CONFLUENCE_OIDC_SCOPES:           "openid,email,profile" | ||||
| CONFLUENCE_OIDC_UNIQUE_ATTRIBUTE: "{{ OIDC.ATTRIBUTES.USERNAME }}" | ||||
|  | ||||
| ## Docker | ||||
| CONFLUENCE_VERSION:               "{{ applications | get_app_conf(application_id, 'docker.services.confluence.version') }}" | ||||
| CONFLUENCE_IMAGE:                 "{{ applications | get_app_conf(application_id, 'docker.services.confluence.image') }}" | ||||
| CONFLUENCE_CONTAINER:             "{{ applications | get_app_conf(application_id, 'docker.services.confluence.name') }}" | ||||
| CONFLUENCE_VERSION:               "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.version') }}" | ||||
| CONFLUENCE_IMAGE:                 "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.image') }}" | ||||
| CONFLUENCE_CONTAINER:             "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.name') }}" | ||||
| CONFLUENCE_DATA_VOLUME:           "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}" | ||||
| CONFLUENCE_CUSTOM_IMAGE:          "{{ CONFLUENCE_IMAGE }}_custom" | ||||
|  | ||||
| ## Performance | ||||
| CONFLUENCE_TOTAL_MB:              "{{ ansible_memtotal_mb | int }}" | ||||
| CONFLUENCE_JVM_MAX_MB:            "{{ [ (CONFLUENCE_TOTAL_MB | int // 2), 12288 ] | min }}" | ||||
| CONFLUENCE_JVM_MIN_MB:            "{{ [ (CONFLUENCE_TOTAL_MB | int // 4), (CONFLUENCE_JVM_MAX_MB | int) ] | min }}" | ||||
| CONFLUENCE_JVM_MIN:               "{{ CONFLUENCE_JVM_MIN_MB }}m" | ||||
| CONFLUENCE_JVM_MAX:               "{{ CONFLUENCE_JVM_MAX_MB }}m" | ||||
|  | ||||
| ## Performance (derive from container limits in config/main.yml) | ||||
| CONFLUENCE_JVM_MAX:               "{{ applications | jvm_max_mb(application_id) }}m" | ||||
| CONFLUENCE_JVM_MIN:               "{{ applications | jvm_min_mb(application_id) }}m" | ||||
|  | ||||
| ## Options  | ||||
| CONFLUENCE_TRUST_STORE_ENABLED:   "{{ applications | get_app_conf(application_id, 'truststore_enabled') }}" | ||||
| @@ -1,6 +1,7 @@ | ||||
| # General | ||||
| application_id:               "web-app-jira" | ||||
| database_type:                "postgres" | ||||
| entity_name:                  "{{ application_id | get_entity_name }}" | ||||
|  | ||||
| # Container | ||||
| container_port:               8080   # Standardport Jira | ||||
| @@ -28,15 +29,12 @@ JIRA_OIDC_SCOPES:             "openid,email,profile" | ||||
| JIRA_OIDC_UNIQUE_ATTRIBUTE:   "{{ OIDC.ATTRIBUTES.USERNAME }}" | ||||
|  | ||||
| ## Docker | ||||
| JIRA_VERSION:                 "{{ applications | get_app_conf(application_id, 'docker.services.jira.version') }}" | ||||
| JIRA_IMAGE:                   "{{ applications | get_app_conf(application_id, 'docker.services.jira.image') }}" | ||||
| JIRA_CONTAINER:               "{{ applications | get_app_conf(application_id, 'docker.services.jira.name') }}" | ||||
| JIRA_VERSION:                 "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.version') }}" | ||||
| JIRA_IMAGE:                   "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.image') }}" | ||||
| JIRA_CONTAINER:               "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.name') }}" | ||||
| JIRA_DATA_VOLUME:             "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}" | ||||
| JIRA_CUSTOM_IMAGE:            "{{ JIRA_IMAGE }}_custom" | ||||
|  | ||||
| ## Performance (auto-derive from host memory) | ||||
| JIRA_TOTAL_MB:                "{{ ansible_memtotal_mb | int }}" | ||||
| JIRA_JVM_MAX_MB:              "{{ [ (JIRA_TOTAL_MB | int // 2), 12288 ] | min }}" | ||||
| JIRA_JVM_MIN_MB:              "{{ [ (JIRA_TOTAL_MB | int // 4), (JIRA_JVM_MAX_MB | int) ] | min }}" | ||||
| JIRA_JVM_MIN:                 "{{ JIRA_JVM_MIN_MB }}m" | ||||
| JIRA_JVM_MAX:                 "{{ JIRA_JVM_MAX_MB }}m" | ||||
| ## Performance (derive from container limits in config/main.yml) | ||||
| JIRA_JVM_MAX:                 "{{ applications | jvm_max_mb(application_id) }}m" | ||||
| JIRA_JVM_MIN:                 "{{ applications | jvm_min_mb(application_id) }}m" | ||||
							
								
								
									
										123
									
								
								tests/unit/filter_plugins/test_jvm_filters.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										123
									
								
								tests/unit/filter_plugins/test_jvm_filters.py
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,123 @@ | ||||
| import unittest | ||||
| from unittest.mock import patch | ||||
|  | ||||
| # Importiere das Filtermodul | ||||
| # Pfad relativ zum Projekt; falls nötig, passe den Importpfad an | ||||
| import importlib | ||||
| jvm_filters = importlib.import_module("filter_plugins.jvm_filters") | ||||
|  | ||||
|  | ||||
| class TestJvmFilters(unittest.TestCase): | ||||
|     def setUp(self): | ||||
|         # Dummy applications dict – Inhalt egal, da get_app_conf gemockt wird | ||||
|         self.apps = {"whatever": True} | ||||
|         self.app_id = "web-app-confluence"  # entity_name wird gemockt | ||||
|  | ||||
|     # ----------------------------- | ||||
|     # Helpers | ||||
|     # ----------------------------- | ||||
|     def _with_conf(self, mem_limit: str, mem_res: str): | ||||
|         """ | ||||
|         Context manager der get_app_conf/get_entity_name passend patched. | ||||
|         """ | ||||
|         patches = [ | ||||
|             patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"), | ||||
|             patch( | ||||
|                 "filter_plugins.jvm_filters.get_app_conf", | ||||
|                 side_effect=lambda apps, app_id, key, required=True: ( | ||||
|                     mem_limit if key.endswith(".mem_limit") | ||||
|                     else mem_res if key.endswith(".mem_reservation") | ||||
|                     else None | ||||
|                 ), | ||||
|             ), | ||||
|         ] | ||||
|         ctxs = [p.start() for p in patches] | ||||
|         self.addCleanup(lambda: [p.stop() for p in patches]) | ||||
|         return ctxs | ||||
|  | ||||
|     # ----------------------------- | ||||
|     # Tests: jvm_max_mb / jvm_min_mb Sizing | ||||
|     # ----------------------------- | ||||
|     def test_sizing_8g_limit_6g_reservation(self): | ||||
|         # mem_limit=8g → candidates: 70% = 5734MB (floor 8*0.7=5.6GB→ 5734MB via int math 8*7//10=5) | ||||
|         # int math: (8*1024)*7//10 = (8192)*7//10 = 5734 | ||||
|         # limit-1024 = 8192-1024 = 7168 | ||||
|         # 12288 | ||||
|         # → Xmx = min(5734, 7168, 12288) = 5734 → floor at 1024 keeps 5734 | ||||
|         # Xms = min(Xmx//2=2867, res=6144, Xmx=5734) = 2867 (>=512) | ||||
|         self._with_conf("8g", "6g") | ||||
|         xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id) | ||||
|         xms = jvm_filters.jvm_min_mb(self.apps, self.app_id) | ||||
|         self.assertEqual(xmx, 5734) | ||||
|         self.assertEqual(xms, 2867) | ||||
|  | ||||
|     def test_sizing_6g_limit_4g_reservation(self): | ||||
|         # limit=6g → 70%: (6144*7)//10 = 4300, limit-1024=5120, 12288 → Xmx=4300 | ||||
|         # Xms=min(4300//2=2150, 4096, 4300)=2150 | ||||
|         self._with_conf("6g", "4g") | ||||
|         xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id) | ||||
|         xms = jvm_filters.jvm_min_mb(self.apps, self.app_id) | ||||
|         self.assertEqual(xmx, 4300) | ||||
|         self.assertEqual(xms, 2150) | ||||
|  | ||||
|     def test_sizing_16g_limit_12g_reservation_cap_12288(self): | ||||
|         # limit=16g → 70%: (16384*7)//10 = 11468, limit-1024=15360, cap=12288 → Xmx=min(11468,15360,12288)=11468 | ||||
|         # Xms=min(11468//2=5734, 12288 (12g), 11468) = 5734 | ||||
|         self._with_conf("16g", "12g") | ||||
|         xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id) | ||||
|         xms = jvm_filters.jvm_min_mb(self.apps, self.app_id) | ||||
|         self.assertEqual(xmx, 11468) | ||||
|         self.assertEqual(xms, 5734) | ||||
|  | ||||
|     def test_floor_small_limit_results_in_min_1024(self): | ||||
|         # limit=1g → 70%: 716, limit-1024=0, 12288 → min=0 → floor → 1024 | ||||
|         self._with_conf("1g", "512m") | ||||
|         xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id) | ||||
|         self.assertEqual(xmx, 1024) | ||||
|  | ||||
|     def test_floor_small_reservation_results_in_min_512(self): | ||||
|         # limit groß genug, aber reservation sehr klein → Xms floored to 512 | ||||
|         self._with_conf("4g", "128m") | ||||
|         xms = jvm_filters.jvm_min_mb(self.apps, self.app_id) | ||||
|         self.assertEqual(xms, 512) | ||||
|  | ||||
|     # ----------------------------- | ||||
|     # Tests: Fehlerfälle / Validierung | ||||
|     # ----------------------------- | ||||
|     def test_invalid_unit_raises(self): | ||||
|         with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"), \ | ||||
|              patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True: | ||||
|                    "8Q" if key.endswith(".mem_limit") else "4g"): | ||||
|             with self.assertRaises(jvm_filters.AnsibleFilterError): | ||||
|                 jvm_filters.jvm_max_mb(self.apps, self.app_id) | ||||
|  | ||||
|     def test_zero_limit_raises(self): | ||||
|         with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"), \ | ||||
|              patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True: | ||||
|                    "0" if key.endswith(".mem_limit") else "4g"): | ||||
|             with self.assertRaises(jvm_filters.AnsibleFilterError): | ||||
|                 jvm_filters.jvm_max_mb(self.apps, self.app_id) | ||||
|  | ||||
|     def test_zero_reservation_raises(self): | ||||
|         with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"), \ | ||||
|              patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True: | ||||
|                    "8g" if key.endswith(".mem_limit") else "0"): | ||||
|             with self.assertRaises(jvm_filters.AnsibleFilterError): | ||||
|                 jvm_filters.jvm_min_mb(self.apps, self.app_id) | ||||
|  | ||||
|     def test_entity_name_is_derived_not_passed(self): | ||||
|         # Sicherstellen, dass get_entity_name() aufgerufen wird und kein externer Parameter nötig ist | ||||
|         with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence") as mock_entity, \ | ||||
|              patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True: | ||||
|                    "8g" if key.endswith(".mem_limit") else "6g"): | ||||
|             xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id) | ||||
|             xms = jvm_filters.jvm_min_mb(self.apps, self.app_id) | ||||
|             self.assertGreater(xmx, 0) | ||||
|             self.assertGreater(xms, 0) | ||||
|             self.assertEqual(mock_entity.call_count, 3) | ||||
|             for call in mock_entity.call_args_list: | ||||
|                 self.assertEqual(call.args[0], self.app_id) | ||||
|  | ||||
|  | ||||
| if __name__ == "__main__": | ||||
|     unittest.main() | ||||
		Reference in New Issue
	
	Block a user