mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2025-10-09 18:28:10 +02:00
feat(jvm): add robust JVM sizing filters and apply across Confluence/Jira
Introduce filter_plugins/jvm_filters.py with jvm_max_mb/jvm_min_mb. Derive Xmx/Xms from docker mem_limit/mem_reservation using safe rules: Xmx=min(70% limit, limit-1024MB, 12288MB), floored at 1024MB; Xms=min(Xmx/2, reservation, Xmx), floored at 512MB. Parse human-readable sizes (k/m/g/t) with binary units. Wire filters into roles: set JVM_MINIMUM_MEMORY/JVM_MAXIMUM_MEMORY via filters; stop relying on host RAM. Keep env templates simple and stable. Add unit tests under tests/unit/filter_plugins/test_jvm_filters.py covering typical sizes, floors, caps, invalid inputs, and entity-name derivation. Ref: https://chatgpt.com/share/68d3b9f6-8d18-800f-aa8d-8a743ddf164d
This commit is contained in:
77
filter_plugins/jvm_filters.py
Normal file
77
filter_plugins/jvm_filters.py
Normal file
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys, os, re
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from ansible.errors import AnsibleFilterError
|
||||
from module_utils.config_utils import get_app_conf
|
||||
from module_utils.entity_name_utils import get_entity_name
|
||||
|
||||
_UNIT_RE = re.compile(r'^\s*(\d+(?:\.\d+)?)\s*([kKmMgGtT]?[bB]?)?\s*$')
|
||||
_FACTORS = {
|
||||
'': 1, 'b': 1,
|
||||
'k': 1024, 'kb': 1024,
|
||||
'm': 1024**2, 'mb': 1024**2,
|
||||
'g': 1024**3, 'gb': 1024**3,
|
||||
't': 1024**4, 'tb': 1024**4,
|
||||
}
|
||||
|
||||
def _to_bytes(v: str) -> int:
|
||||
if v is None:
|
||||
raise AnsibleFilterError("jvm_filters: size value is None")
|
||||
s = str(v).strip()
|
||||
m = _UNIT_RE.match(s)
|
||||
if not m:
|
||||
raise AnsibleFilterError(f"jvm_filters: invalid size '{v}'")
|
||||
num, unit = m.group(1), (m.group(2) or '').lower()
|
||||
try:
|
||||
val = float(num)
|
||||
except ValueError as e:
|
||||
raise AnsibleFilterError(f"jvm_filters: invalid numeric size '{v}'") from e
|
||||
factor = _FACTORS.get(unit)
|
||||
if factor is None:
|
||||
raise AnsibleFilterError(f"jvm_filters: unknown unit in '{v}'")
|
||||
return int(val * factor)
|
||||
|
||||
def _to_mb(v: str) -> int:
|
||||
return max(0, _to_bytes(v) // (1024 * 1024))
|
||||
|
||||
def _svc(app_id: str) -> str:
|
||||
return get_entity_name(app_id)
|
||||
|
||||
def _mem_limit_mb(apps: dict, app_id: str) -> int:
|
||||
svc = _svc(app_id)
|
||||
raw = get_app_conf(apps, app_id, f"docker.services.{svc}.mem_limit")
|
||||
mb = _to_mb(raw)
|
||||
if mb <= 0:
|
||||
raise AnsibleFilterError(f"jvm_filters: mem_limit for '{svc}' must be > 0 MB (got '{raw}')")
|
||||
return mb
|
||||
|
||||
def _mem_res_mb(apps: dict, app_id: str) -> int:
|
||||
svc = _svc(app_id)
|
||||
raw = get_app_conf(apps, app_id, f"docker.services.{svc}.mem_reservation")
|
||||
mb = _to_mb(raw)
|
||||
if mb <= 0:
|
||||
raise AnsibleFilterError(f"jvm_filters: mem_reservation for '{svc}' must be > 0 MB (got '{raw}')")
|
||||
return mb
|
||||
|
||||
def jvm_max_mb(apps: dict, app_id: str) -> int:
|
||||
"""Xmx = min( floor(0.7*limit), limit-1024, 12288 ) with floor at 1024 MB."""
|
||||
limit_mb = _mem_limit_mb(apps, app_id)
|
||||
c1 = (limit_mb * 7) // 10
|
||||
c2 = max(0, limit_mb - 1024)
|
||||
c3 = 12288
|
||||
return max(1024, min(c1, c2, c3))
|
||||
|
||||
def jvm_min_mb(apps: dict, app_id: str) -> int:
|
||||
"""Xms = min( floor(Xmx/2), mem_reservation, Xmx ) with floor at 512 MB."""
|
||||
xmx = jvm_max_mb(apps, app_id)
|
||||
res = _mem_res_mb(apps, app_id)
|
||||
return max(512, min(xmx // 2, res, xmx))
|
||||
|
||||
class FilterModule(object):
|
||||
def filters(self):
|
||||
return {
|
||||
"jvm_max_mb": jvm_max_mb,
|
||||
"jvm_min_mb": jvm_min_mb,
|
||||
}
|
@@ -1,6 +1,7 @@
|
||||
# General
|
||||
application_id: "web-app-confluence"
|
||||
database_type: "postgres"
|
||||
entity_name: "{{ application_id | get_entity_name }}"
|
||||
|
||||
# Container
|
||||
container_port: 8090
|
||||
@@ -28,19 +29,15 @@ CONFLUENCE_OIDC_SCOPES: "openid,email,profile"
|
||||
CONFLUENCE_OIDC_UNIQUE_ATTRIBUTE: "{{ OIDC.ATTRIBUTES.USERNAME }}"
|
||||
|
||||
## Docker
|
||||
CONFLUENCE_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.confluence.version') }}"
|
||||
CONFLUENCE_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.confluence.image') }}"
|
||||
CONFLUENCE_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.confluence.name') }}"
|
||||
CONFLUENCE_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.version') }}"
|
||||
CONFLUENCE_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.image') }}"
|
||||
CONFLUENCE_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.name') }}"
|
||||
CONFLUENCE_DATA_VOLUME: "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}"
|
||||
CONFLUENCE_CUSTOM_IMAGE: "{{ CONFLUENCE_IMAGE }}_custom"
|
||||
|
||||
## Performance
|
||||
CONFLUENCE_TOTAL_MB: "{{ ansible_memtotal_mb | int }}"
|
||||
CONFLUENCE_JVM_MAX_MB: "{{ [ (CONFLUENCE_TOTAL_MB | int // 2), 12288 ] | min }}"
|
||||
CONFLUENCE_JVM_MIN_MB: "{{ [ (CONFLUENCE_TOTAL_MB | int // 4), (CONFLUENCE_JVM_MAX_MB | int) ] | min }}"
|
||||
CONFLUENCE_JVM_MIN: "{{ CONFLUENCE_JVM_MIN_MB }}m"
|
||||
CONFLUENCE_JVM_MAX: "{{ CONFLUENCE_JVM_MAX_MB }}m"
|
||||
|
||||
## Performance (derive from container limits in config/main.yml)
|
||||
CONFLUENCE_JVM_MAX: "{{ applications | jvm_max_mb(application_id) }}m"
|
||||
CONFLUENCE_JVM_MIN: "{{ applications | jvm_min_mb(application_id) }}m"
|
||||
|
||||
## Options
|
||||
CONFLUENCE_TRUST_STORE_ENABLED: "{{ applications | get_app_conf(application_id, 'truststore_enabled') }}"
|
@@ -1,6 +1,7 @@
|
||||
# General
|
||||
application_id: "web-app-jira"
|
||||
database_type: "postgres"
|
||||
entity_name: "{{ application_id | get_entity_name }}"
|
||||
|
||||
# Container
|
||||
container_port: 8080 # Standardport Jira
|
||||
@@ -28,15 +29,12 @@ JIRA_OIDC_SCOPES: "openid,email,profile"
|
||||
JIRA_OIDC_UNIQUE_ATTRIBUTE: "{{ OIDC.ATTRIBUTES.USERNAME }}"
|
||||
|
||||
## Docker
|
||||
JIRA_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.jira.version') }}"
|
||||
JIRA_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.jira.image') }}"
|
||||
JIRA_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.jira.name') }}"
|
||||
JIRA_VERSION: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.version') }}"
|
||||
JIRA_IMAGE: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.image') }}"
|
||||
JIRA_CONTAINER: "{{ applications | get_app_conf(application_id, 'docker.services.' ~ entity_name ~ '.name') }}"
|
||||
JIRA_DATA_VOLUME: "{{ applications | get_app_conf(application_id, 'docker.volumes.data') }}"
|
||||
JIRA_CUSTOM_IMAGE: "{{ JIRA_IMAGE }}_custom"
|
||||
|
||||
## Performance (auto-derive from host memory)
|
||||
JIRA_TOTAL_MB: "{{ ansible_memtotal_mb | int }}"
|
||||
JIRA_JVM_MAX_MB: "{{ [ (JIRA_TOTAL_MB | int // 2), 12288 ] | min }}"
|
||||
JIRA_JVM_MIN_MB: "{{ [ (JIRA_TOTAL_MB | int // 4), (JIRA_JVM_MAX_MB | int) ] | min }}"
|
||||
JIRA_JVM_MIN: "{{ JIRA_JVM_MIN_MB }}m"
|
||||
JIRA_JVM_MAX: "{{ JIRA_JVM_MAX_MB }}m"
|
||||
## Performance (derive from container limits in config/main.yml)
|
||||
JIRA_JVM_MAX: "{{ applications | jvm_max_mb(application_id) }}m"
|
||||
JIRA_JVM_MIN: "{{ applications | jvm_min_mb(application_id) }}m"
|
123
tests/unit/filter_plugins/test_jvm_filters.py
Normal file
123
tests/unit/filter_plugins/test_jvm_filters.py
Normal file
@@ -0,0 +1,123 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
# Importiere das Filtermodul
|
||||
# Pfad relativ zum Projekt; falls nötig, passe den Importpfad an
|
||||
import importlib
|
||||
jvm_filters = importlib.import_module("filter_plugins.jvm_filters")
|
||||
|
||||
|
||||
class TestJvmFilters(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# Dummy applications dict – Inhalt egal, da get_app_conf gemockt wird
|
||||
self.apps = {"whatever": True}
|
||||
self.app_id = "web-app-confluence" # entity_name wird gemockt
|
||||
|
||||
# -----------------------------
|
||||
# Helpers
|
||||
# -----------------------------
|
||||
def _with_conf(self, mem_limit: str, mem_res: str):
|
||||
"""
|
||||
Context manager der get_app_conf/get_entity_name passend patched.
|
||||
"""
|
||||
patches = [
|
||||
patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"),
|
||||
patch(
|
||||
"filter_plugins.jvm_filters.get_app_conf",
|
||||
side_effect=lambda apps, app_id, key, required=True: (
|
||||
mem_limit if key.endswith(".mem_limit")
|
||||
else mem_res if key.endswith(".mem_reservation")
|
||||
else None
|
||||
),
|
||||
),
|
||||
]
|
||||
ctxs = [p.start() for p in patches]
|
||||
self.addCleanup(lambda: [p.stop() for p in patches])
|
||||
return ctxs
|
||||
|
||||
# -----------------------------
|
||||
# Tests: jvm_max_mb / jvm_min_mb Sizing
|
||||
# -----------------------------
|
||||
def test_sizing_8g_limit_6g_reservation(self):
|
||||
# mem_limit=8g → candidates: 70% = 5734MB (floor 8*0.7=5.6GB→ 5734MB via int math 8*7//10=5)
|
||||
# int math: (8*1024)*7//10 = (8192)*7//10 = 5734
|
||||
# limit-1024 = 8192-1024 = 7168
|
||||
# 12288
|
||||
# → Xmx = min(5734, 7168, 12288) = 5734 → floor at 1024 keeps 5734
|
||||
# Xms = min(Xmx//2=2867, res=6144, Xmx=5734) = 2867 (>=512)
|
||||
self._with_conf("8g", "6g")
|
||||
xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
xms = jvm_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
self.assertEqual(xmx, 5734)
|
||||
self.assertEqual(xms, 2867)
|
||||
|
||||
def test_sizing_6g_limit_4g_reservation(self):
|
||||
# limit=6g → 70%: (6144*7)//10 = 4300, limit-1024=5120, 12288 → Xmx=4300
|
||||
# Xms=min(4300//2=2150, 4096, 4300)=2150
|
||||
self._with_conf("6g", "4g")
|
||||
xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
xms = jvm_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
self.assertEqual(xmx, 4300)
|
||||
self.assertEqual(xms, 2150)
|
||||
|
||||
def test_sizing_16g_limit_12g_reservation_cap_12288(self):
|
||||
# limit=16g → 70%: (16384*7)//10 = 11468, limit-1024=15360, cap=12288 → Xmx=min(11468,15360,12288)=11468
|
||||
# Xms=min(11468//2=5734, 12288 (12g), 11468) = 5734
|
||||
self._with_conf("16g", "12g")
|
||||
xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
xms = jvm_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
self.assertEqual(xmx, 11468)
|
||||
self.assertEqual(xms, 5734)
|
||||
|
||||
def test_floor_small_limit_results_in_min_1024(self):
|
||||
# limit=1g → 70%: 716, limit-1024=0, 12288 → min=0 → floor → 1024
|
||||
self._with_conf("1g", "512m")
|
||||
xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
self.assertEqual(xmx, 1024)
|
||||
|
||||
def test_floor_small_reservation_results_in_min_512(self):
|
||||
# limit groß genug, aber reservation sehr klein → Xms floored to 512
|
||||
self._with_conf("4g", "128m")
|
||||
xms = jvm_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
self.assertEqual(xms, 512)
|
||||
|
||||
# -----------------------------
|
||||
# Tests: Fehlerfälle / Validierung
|
||||
# -----------------------------
|
||||
def test_invalid_unit_raises(self):
|
||||
with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"), \
|
||||
patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True:
|
||||
"8Q" if key.endswith(".mem_limit") else "4g"):
|
||||
with self.assertRaises(jvm_filters.AnsibleFilterError):
|
||||
jvm_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
|
||||
def test_zero_limit_raises(self):
|
||||
with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"), \
|
||||
patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True:
|
||||
"0" if key.endswith(".mem_limit") else "4g"):
|
||||
with self.assertRaises(jvm_filters.AnsibleFilterError):
|
||||
jvm_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
|
||||
def test_zero_reservation_raises(self):
|
||||
with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence"), \
|
||||
patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True:
|
||||
"8g" if key.endswith(".mem_limit") else "0"):
|
||||
with self.assertRaises(jvm_filters.AnsibleFilterError):
|
||||
jvm_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
|
||||
def test_entity_name_is_derived_not_passed(self):
|
||||
# Sicherstellen, dass get_entity_name() aufgerufen wird und kein externer Parameter nötig ist
|
||||
with patch("filter_plugins.jvm_filters.get_entity_name", return_value="confluence") as mock_entity, \
|
||||
patch("filter_plugins.jvm_filters.get_app_conf", side_effect=lambda apps, app_id, key, required=True:
|
||||
"8g" if key.endswith(".mem_limit") else "6g"):
|
||||
xmx = jvm_filters.jvm_max_mb(self.apps, self.app_id)
|
||||
xms = jvm_filters.jvm_min_mb(self.apps, self.app_id)
|
||||
self.assertGreater(xmx, 0)
|
||||
self.assertGreater(xms, 0)
|
||||
self.assertEqual(mock_entity.call_count, 3)
|
||||
for call in mock_entity.call_args_list:
|
||||
self.assertEqual(call.args[0], self.app_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
Reference in New Issue
Block a user