kevinveenbirkenbach/computer-playbook
1
0 Fork 0
mirror of https://github.com/kevinveenbirkenbach/computer-playbook.git synced 2026-11-09 12:17:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feature/store-i18n

Browse Source 
Three conflicts, all from main sizing memory parameters off the container
limit while this branch moved container environments into one file per role.

plugins/filter/memory_filters.py and its test are deleted on main, replaced
by the service_memory_mb lookup. This branch only renamed the filter out of
its get_ prefix and fixed the docstrings' reStructuredText, so nothing of
substance is lost; the deletion is taken. Verified first that no consumer
survives: jvm_max_mb, jvm_min_mb and redis_maxmemory_mb appear nowhere in
roles/, group_vars/, tests/, plugins/, cli/ or utils/ once the merged tree
is in place.

web-app-zammad's search service is the only real conflict. 37e80b957e moved
discovery.type, xpack.security.enabled and ES_JAVA_OPTS from the compose
template into templates/env.j2, where an inline `environment:` block is what
the compose-environment-from-env-file lint rejects; a7db89ac0b changed the
heap from a literal 512m to half the container's limit. Both intents hold,
so the keys stay in env.j2 and the heap now reads ZAMMAD_SEARCH_HEAP_MB,
which the merged vars/main.yml already defines through the new lookup.

web-app-magento and web-app-shopware keep their inline environment blocks:
both carry an explicit nocheck naming the reason, that their single env.j2
is shared with containers the search daemon's keys must not reach.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
kevinveenbirkenbachandClaude Opus 5 committed 2026-10-07 00:57:07 +02:00
2 parents a1975c1094 a7db89ac0b
commit 774ff8c2d2
34 files changed
+248 -461
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

plugins/filter/memory_filters.py
-188
View File
@@ -1,188 +0,0 @@
from __future__ import annotations
import re
from ansible.errors import AnsibleFilterError
from utils.roles.applications.config import get
from utils.roles.entity.name import entity_name
_UNIT_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([kKmMgGtT]?[bB]?)?\s*$")
_FACTORS = {
"": 1,
"b": 1,
"k": 1024,
"kb": 1024,
"m": 1024**2,
"mb": 1024**2,
"g": 1024**3,
"gb": 1024**3,
"t": 1024**4,
"tb": 1024**4,
}
# ------------------------------------------------------
# Helpers: unit conversion
# ------------------------------------------------------
def _to_bytes(v: str) -> int:
"""Convert a human-readable size string (e.g., '2g', '512m') to bytes."""
if v is None:
raise AnsibleFilterError("memory_filters: size value is None")
s = str(v).strip()
m = _UNIT_RE.match(s)
if not m:
raise AnsibleFilterError(f"memory_filters: invalid size '{v}'")
num, unit = m.group(1), (m.group(2) or "").lower()
try:
val = float(num)
except ValueError as e:
raise AnsibleFilterError(f"memory_filters: invalid numeric size '{v}'") from e
factor = _FACTORS.get(unit)
if factor is None:
raise AnsibleFilterError(f"memory_filters: unknown unit in '{v}'")
return int(val * factor)
def _to_mb(v: str) -> int:
"""Convert human-readable size to megabytes."""
return max(0, _to_bytes(v) // (1024 * 1024))
# ------------------------------------------------------
# JVM-specific helpers
# ------------------------------------------------------
def _svc(app_id: str) -> str:
"""Resolve the internal service name for JVM-based applications."""
return entity_name(app_id)
def _mem_limit_mb(apps: dict, app_id: str) -> int:
"""Resolve mem_limit for the JVM service of the given application."""
svc = _svc(app_id)
raw = get(apps, app_id, f"services.{svc}.mem_limit")
mb = _to_mb(raw)
if mb <= 0:
raise AnsibleFilterError(
f"memory_filters: mem_limit for '{svc}' must be > 0 MB (got '{raw}')"
)
return mb
def _mem_res_mb(apps: dict, app_id: str) -> int:
"""Resolve mem_reservation for the JVM service of the given application."""
svc = _svc(app_id)
raw = get(apps, app_id, f"services.{svc}.mem_reservation")
mb = _to_mb(raw)
if mb <= 0:
raise AnsibleFilterError(
f"memory_filters: mem_reservation for '{svc}' must be > 0 MB (got '{raw}')"
)
return mb
def jvm_max_mb(apps: dict, app_id: str) -> int:
"""
Compute recommended JVM Xmx in MB using::
Xmx = min(
floor(0.7 * mem_limit),
mem_limit - 1024,
12288
)
with a lower bound of 1024 MB.
"""
limit_mb = _mem_limit_mb(apps, app_id)
c1 = (limit_mb * 7) // 10
c2 = max(0, limit_mb - 1024)
c3 = 12288
return max(1024, min(c1, c2, c3))
def jvm_min_mb(apps: dict, app_id: str) -> int:
"""
Compute recommended JVM Xms in MB using::
Xms = min(
floor(Xmx / 2),
mem_reservation,
Xmx
)
with a lower bound of 512 MB.
"""
xmx = jvm_max_mb(apps, app_id)
res = _mem_res_mb(apps, app_id)
return max(512, min(xmx // 2, res, xmx))
# ------------------------------------------------------
# Redis-specific helpers (always service name "redis")
# ------------------------------------------------------
def _redis_mem_limit_mb(apps: dict, app_id: str, default_mb: int = 256) -> int:
"""
Resolve mem_limit for the Redis service of an application.
Unlike JVM-based services, Redis always uses the service name "redis".
If no mem_limit is defined, fall back to default_mb.
"""
raw = get(
apps,
app_id,
"services.redis.mem_limit",
strict=False,
default=f"{default_mb}m",
)
mb = _to_mb(raw)
if mb <= 0:
raise AnsibleFilterError(
f"memory_filters: mem_limit for 'redis' must be > 0 MB (got '{raw}')"
)
return mb
def redis_maxmemory_mb(
apps: dict, app_id: str, factor: float = 0.8, min_mb: int = 64
) -> int:
"""
Compute recommended Redis `maxmemory` in MB.
* factor: fraction of allowed memory used for Redis data (default 0.8)
* min_mb: minimum floor value (default 64 MB)
maxmemory = max(min_mb, floor(factor * mem_limit))
"""
limit_mb = _redis_mem_limit_mb(apps, app_id)
return max(min_mb, int(limit_mb * factor))
# ------------------------------------------------------
# Filter module
# ------------------------------------------------------
class FilterModule:
def filters(self):
return {
"jvm_max_mb": jvm_max_mb,
"jvm_min_mb": jvm_min_mb,
"redis_maxmemory_mb": redis_maxmemory_mb,
}
plugins/lookup/service_memory_mb.py
+106
View File
@@ -0,0 +1,106 @@
"""Derive a memory parameter in MiB from the memory limit of a service's container.
-Xmx{{ lookup('service_memory_mb', application_id, 'search', 0.5) }}m
memory_limit = {{ lookup('service_memory_mb', application_id, entity_name, 0.5) }}M
The limit is resolved the way ``sys-svc-container`` resolves the container's own
``mem_limit``: ``services.<service>.mem_limit``, then the role's primary entity,
then ``RESOURCE_MEM_LIMIT``. ``share`` is the fraction of that limit the
parameter may claim, so a heap, a cache or a per-request limit follows the
container instead of a literal that drifts away from it.
Sizes are read like Docker reads them: ``k``, ``m`` and ``g`` are binary, a bare
number is bytes. The result is whole MiB, the unit ``-Xmx<n>m``, PHP ``<n>M``,
``memcached -m``, Redis ``<n>mb`` and PostgreSQL ``<n>MB`` all share.
"""
from __future__ import annotations
import re
from typing import Any
from ansible.errors import AnsibleError
from ansible.plugins.loader import lookup_loader
from ansible.plugins.lookup import LookupBase
from plugins.filter.resource_filter import resource_filter
_SIZE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([bkmg]?)b?\s*$", re.IGNORECASE)
_FACTOR = {"": 1, "b": 1, "k": 1024, "m": 1024**2, "g": 1024**3}
_MEBIBYTE = 1024**2
_HOST_DEFAULT = "RESOURCE_MEM_LIMIT"
_UNDECLARED = object()
def docker_bytes(size: Any) -> int:
"""Return a Docker memory size in bytes.
Args:
size: a size such as ``2g``, ``1.5g``, ``512m`` or a byte count.
Raises:
AnsibleError: the size is not a number with an optional b/k/m/g unit.
"""
match = _SIZE.match(str(size))
if match is None:
raise AnsibleError(f"lookup('service_memory_mb'): unreadable size {size!r}")
return int(float(match.group(1)) * _FACTOR[match.group(2).lower()])
def share_mb(mem_limit: Any, share: Any) -> int:
"""Return ``share`` of ``mem_limit`` in whole MiB.
Args:
mem_limit: Docker size of the container limit, e.g. ``2g`` or ``512m``.
share: fraction of the limit, above 0 and at most 1.
Raises:
AnsibleError: the share is no fraction in that range, the size is
unreadable, or the result is below 1 MiB.
"""
try:
fraction = float(share)
except (TypeError, ValueError) as exc:
raise AnsibleError(
f"lookup('service_memory_mb'): share {share!r} is not a number"
) from exc
if not 0 < fraction <= 1:
raise AnsibleError(
f"lookup('service_memory_mb'): share {share!r} must be above 0 and at most 1"
)
megabytes = int(docker_bytes(mem_limit) * fraction) // _MEBIBYTE
if megabytes < 1:
raise AnsibleError(
f"lookup('service_memory_mb'): {share!r} of {mem_limit!r} is below 1 MiB"
)
return megabytes
class LookupModule(LookupBase):
"""lookup('service_memory_mb', application_id, service_name, share)"""
def run(self, terms, variables: dict[str, Any] | None = None, **kwargs):
if len(terms) != 3:
raise AnsibleError(
"lookup('service_memory_mb', application_id, service_name, share) "
"expects exactly 3 positional terms."
)
application_id, service_name, share = terms
templar = self._templar
variables = variables or templar.available_variables
applications = lookup_loader.get(
"applications", loader=self._loader, templar=templar
).run([], variables=variables)[0]
mem_limit = resource_filter(
applications, application_id, "mem_limit", service_name, _UNDECLARED
)
if mem_limit is _UNDECLARED:
if _HOST_DEFAULT not in variables:
raise AnsibleError(
f"lookup('service_memory_mb'): '{application_id}' declares no "
f"mem_limit for '{service_name}' and {_HOST_DEFAULT} is unset"
)
mem_limit = templar.template(variables[_HOST_DEFAULT])
return [share_mb(mem_limit, share)]
roles/svc-db-elasticsearch/templates/env.j2
+1 -1
View File
@@ -6,4 +6,4 @@ xpack.security.enabled=true
xpack.security.http.ssl.enabled=false
xpack.security.transport.ssl.enabled=false
bootstrap.memory_lock=true
ES_JAVA_OPTS=-Xms1g -Xmx1g
ES_JAVA_OPTS=-Xms{{ ELASTICSEARCH_HEAP_MB }}m -Xmx{{ ELASTICSEARCH_HEAP_MB }}m
roles/svc-db-elasticsearch/vars/main.yml
+1
View File
@@ -10,6 +10,7 @@ ELASTICSEARCH_PASSWORD: "{{ lookup('config', application_id, 'secrets.credential
ELASTICSEARCH_EXPOSE_LOCAL: true
ELASTICSEARCH_LOCAL_HOST: "127.0.0.1"
ELASTICSEARCH_INTERNAL_PORT: "{{ lookup('config', application_id, 'services.elasticsearch.ports.internal.elasticsearch') }}"
ELASTICSEARCH_HEAP_MB: "{{ lookup('service_memory_mb', application_id, entity_name, 0.5) }}"
ELASTICSEARCH_RETRIES: 5
ELASTICSEARCH_DELAY: 2
ELASTICSEARCH_API_BASE: "http://{{ ELASTICSEARCH_LOCAL_HOST }}:{{ ELASTICSEARCH_INTERNAL_PORT }}"
roles/svc-db-memcached/templates/compose.yml.j2
+1 -4
View File
@@ -3,10 +3,7 @@
{{ service_name }}:
{{ lookup('compose_only', 'container_name', MEMCACHED_CONTAINER) }}
{{ lookup('container_image', application_id, 'memcached') }}
command:
- "memcached"
- "-m"
- "{{ MEMCACHED_MEMORY_LIMIT_MB }}"
command: ["memcached", "-m", "{{ MEMCACHED_MEMORY_LIMIT_MB }}"]
{% include 'roles/sys-svc-container/templates/base.yml.j2' %}
{% if MEMCACHED_EXPOSE_LOCAL %}
roles/svc-db-memcached/vars/main.yml
+1 -1
View File
@@ -10,6 +10,6 @@ MEMCACHED_EXPOSE_LOCAL: true
MEMCACHED_LOCAL_HOST: "127.0.0.1"
MEMCACHED_RETRIES: 5
MEMCACHED_DELAY: 5
MEMCACHED_MEMORY_LIMIT_MB: 192
MEMCACHED_MEMORY_LIMIT_MB: "{{ lookup('service_memory_mb', application_id, entity_name, 0.75) }}"
MEMCACHED_EXEC_SHELL: "container exec {{ MEMCACHED_CONTAINER_ADDRESS }} sh -lc"
MEMCACHED_PROBE: "printf \"version\\r\\n\" | nc -w 1 {{ MEMCACHED_LOCAL_HOST }} 11211 | grep -q VERSION"
roles/svc-db-redis/templates/compose.yml.j2
+1 -1
View File
@@ -10,7 +10,7 @@
exec redis-server
--requirepass {{ REDIS_PASSWORD | quote }}
--appendonly yes
--maxmemory {{ lookup('applications') | redis_maxmemory_mb(application_id, 0.8, RESOURCE_MEM_LIMIT | int) }}mb
--maxmemory {{ lookup('service_memory_mb', application_id, service_name, 0.8) }}mb
--maxmemory-policy allkeys-lru
{% include 'roles/sys-svc-container/templates/base.yml.j2' %}
roles/svc-db-redis/templates/service.yml.j2
+1 -1
View File
@@ -13,7 +13,7 @@
- "no"
- --save
- ""
- --maxmemory {{ lookup('applications') | redis_maxmemory_mb(application_id, 0.8, RESOURCE_MEM_LIMIT | int ) }}mb
- --maxmemory {{ lookup('service_memory_mb', application_id, 'redis', 0.8) }}mb
- --maxmemory-policy
- "allkeys-lru"
# nocheck: container-healthcheck-literal probes every second, so the SPOT's 5s start cadence would report healthy later than this block already does
roles/web-app-confluence/vars/main.yml
+2 -2
View File
@@ -25,7 +25,7 @@ CONFLUENCE_VERSION: "{{ lookup('config', application_id, 'services.' ~ entity_na
CONFLUENCE_IMAGE: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.image') }}"
CONFLUENCE_CONTAINER: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.name') }}"
CONFLUENCE_JVM_MAX: "{{ lookup('applications') | jvm_max_mb(application_id) }}m"
CONFLUENCE_JVM_MIN: "{{ lookup('applications') | jvm_min_mb(application_id) }}m"
CONFLUENCE_JVM_MAX: "{{ lookup('service_memory_mb', application_id, entity_name, 0.7) }}m"
CONFLUENCE_JVM_MIN: "{{ lookup('service_memory_mb', application_id, entity_name, 0.35) }}m"
CONFLUENCE_TRUST_STORE_ENABLED: "{{ lookup('config', application_id, 'services.confluence.truststore_enabled') }}"
roles/web-app-jira/vars/main.yml
+2 -2
View File
@@ -25,5 +25,5 @@ JIRA_VERSION: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '
JIRA_IMAGE: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.image') }}"
JIRA_CONTAINER: "{{ lookup('config', application_id, 'services.' ~ entity_name ~ '.name') }}"
JIRA_JVM_MAX: "{{ lookup('applications') | jvm_max_mb(application_id) }}m"
JIRA_JVM_MIN: "{{ lookup('applications') | jvm_min_mb(application_id) }}m"
JIRA_JVM_MAX: "{{ lookup('service_memory_mb', application_id, entity_name, 0.7) }}m"
JIRA_JVM_MIN: "{{ lookup('service_memory_mb', application_id, entity_name, 0.35) }}m"
roles/web-app-joomla/meta/services.yml
-3
View File
@@ -51,9 +51,6 @@ joomla:
no_stop_required: true
upload:
size_mb: 100
performance:
php:
memory_limit: 512M
autocreate_users: true
ports:
local:
roles/web-app-joomla/tasks/08_oidc_plugin.yml
+4 -4
View File
@@ -53,7 +53,7 @@
- name: "📦 Install plg_system_keycloak via Joomla CLI"
ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node
container exec --user {{ JOOMLA_WEB_USER | quote }} {{ JOOMLA_LOCAL_CID | quote }}
php -d memory_limit=512M {{ JOOMLA_CLI_FILE | quote }}
php -d memory_limit={{ JOOMLA_PHP_MEMORY_LIMIT }} {{ JOOMLA_CLI_FILE | quote }}
extension:install --path={{ JOOMLA_OIDC_PLUGIN_ZIP | quote }}
register: plugin_install
changed_when: "'Extension installed successfully' in plugin_install.stdout"
@@ -63,8 +63,8 @@
delegate_to: "{{ JOOMLA_HOST_NODE }}"
- name: "🔍 Resolve plg_system_keycloak extension ID" # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node
ansible.builtin.shell: "container exec --user {{ JOOMLA_WEB_USER }} {{ JOOMLA_LOCAL_CID | quote }} php -d memory_limit=512M {{ JOOMLA_CLI_FILE }} extension:list
| awk '/plg_system_keycloak/ {print $2; exit}'"
ansible.builtin.shell: "container exec --user {{ JOOMLA_WEB_USER }} {{ JOOMLA_LOCAL_CID | quote }} php -d memory_limit={{ JOOMLA_PHP_MEMORY_LIMIT }} {{ JOOMLA_CLI_FILE
}} extension:list | awk '/plg_system_keycloak/ {print $2; exit}'"
register: plugin_id_lookup
changed_when: false
failed_when: plugin_id_lookup.stdout | trim | length == 0
@@ -73,7 +73,7 @@
- name: "🟢 Enable plg_system_keycloak via Joomla CLI"
ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node
container exec --user {{ JOOMLA_WEB_USER | quote }} {{ JOOMLA_LOCAL_CID | quote }}
php -d memory_limit=512M {{ JOOMLA_CLI_FILE | quote }}
php -d memory_limit={{ JOOMLA_PHP_MEMORY_LIMIT }} {{ JOOMLA_CLI_FILE | quote }}
extension:enable {{ plugin_id_lookup.stdout | trim | quote }}
register: plugin_enable
changed_when: "'has been enabled' in plugin_enable.stdout or 'enabled successfully' in plugin_enable.stdout"
roles/web-app-joomla/vars/main.yml
+1 -1
View File
@@ -23,7 +23,7 @@ JOOMLA_UPLOAD_CONFIG: "{{ [ lookup('container', application_id, 'directories.ins
JOOMLA_UPLOAD_SIZE: "{{ lookup('config', application_id, 'services.joomla.upload.size_mb') }}"
JOOMLA_UPLOAD_MAX_FILESIZE: "{{ (JOOMLA_UPLOAD_SIZE | int) }}M"
JOOMLA_POST_MAX_SIZE: "{{ ((JOOMLA_UPLOAD_SIZE | int) * 12 // 10) }}M"
JOOMLA_PHP_MEMORY_LIMIT: "{{ lookup('config', application_id, 'services.joomla.performance.php.memory_limit') }}"
JOOMLA_PHP_MEMORY_LIMIT: "{{ lookup('service_memory_mb', application_id, entity_name, 0.5) }}M"
JOOMLA_USER_NAME: "{{ lookup('users', 'administrator').username }}"
JOOMLA_USER: "{{ JOOMLA_USER_NAME | capitalize }}"
roles/web-app-magento/templates/compose.yml.j2
+1 -1
View File
@@ -39,7 +39,7 @@
environment: # nocheck: compose-environment-from-env-file search-only keys; this role sets no docker_compose_env, so its single templates/env.j2 is also the env_file of the nginx and php containers, which must not receive the OpenSearch daemon's discovery.type, plugins.security.disabled or OPENSEARCH_JAVA_OPTS
- discovery.type=single-node
- plugins.security.disabled=true
- OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m
- OPENSEARCH_JAVA_OPTS=-Xms{{ MAGENTO_SEARCH_HEAP_MB }}m -Xmx{{ MAGENTO_SEARCH_HEAP_MB }}m
{{ lookup('container_healthcheck', service_name) | indent(4) }}
{{ lookup('container_extra_hosts') | indent(4) }}
{{ lookup('container_networks') | indent(4) }}
roles/web-app-magento/templates/env.j2
+1 -1
View File
@@ -41,6 +41,6 @@ MAGENTO_REMOTE_STORAGE_REGION={{ lookup('objstore', application_id, 'region') }}
MAGENTO_REMOTE_STORAGE_PATH_STYLE=1
{% endif %}
PHP_MEMORY_LIMIT=768M
PHP_MEMORY_LIMIT={{ lookup('service_memory_mb', application_id, 'php', 0.75) }}M
APACHE_SERVERNAME={{ MAGENTO_DOMAIN }}
roles/web-app-magento/vars/main.yml
+1
View File
@@ -19,6 +19,7 @@ MAGENTO_PHP_FPM_CONF_PATH: "{{ [ lookup('container', application_id, 'directorie
MAGENTO_OBJSTORE_ENABLED: "{{ lookup('objstore', application_id, 'enabled') }}"
MAGENTO_SEARCH_CONTAINER: "{{ lookup('config', application_id, 'services.search.name') }}"
MAGENTO_SEARCH_HEAP_MB: "{{ lookup('service_memory_mb', application_id, 'search', 0.5) }}"
MAGENTO_PHP_CONTAINER: "{{ lookup('config', application_id, 'services.php.name') }}"
MAGENTO_PHP_CONTAINER_ADDRESS: "{{ lookup('container_address', application_id, 'php') }}"
roles/web-app-moodle/files/build/php-conf/opcache-recommended.ini
-1
View File
@@ -1,4 +1,3 @@
opcache.memory_consumption=256
opcache.interned_strings_buffer=16
opcache.max_accelerated_files=20000
opcache.revalidate_freq=60
roles/web-app-moodle/meta/services.yml
-1
View File
@@ -63,7 +63,6 @@ moodle:
pids_limit: 512
performance:
php:
memory_limit: 512M
upload_max_filesize: 256M
post_max_size: 256M
ports:
roles/web-app-moodle/templates/build/php-conf/moodle-runtime.ini.j2
+1
View File
@@ -1,4 +1,5 @@
memory_limit={{ MOODLE_PHP_MEMORY_LIMIT }}
opcache.memory_consumption={{ lookup('service_memory_mb', application_id, 'moodle', 0.125) }}
upload_max_filesize={{ MOODLE_PHP_UPLOAD_MAX_FILESIZE }}
post_max_size={{ MOODLE_PHP_POST_MAX_SIZE }}
max_execution_time={{ MOODLE_PHP_MAX_EXECUTION_TIME }}
roles/web-app-moodle/vars/main.yml
+1 -1
View File
@@ -144,7 +144,7 @@ MOODLE_SITE_TITLE: "{{ lookup('design', application_id).label }}"
MOODLE_FPM_PORT: "{{ lookup('config', application_id, 'services.moodle.ports.internal.fpm') }}"
MOODLE_NGINX_PORT: "{{ lookup('config', application_id, 'services.moodle.ports.internal.http') }}"
MOODLE_PHP_MEMORY_LIMIT: "{{ lookup('config', application_id, 'services.moodle.performance.php.memory_limit') }}"
MOODLE_PHP_MEMORY_LIMIT: "{{ lookup('service_memory_mb', application_id, 'moodle', 0.25) }}M"
MOODLE_PHP_UPLOAD_MAX_FILESIZE: "{{ lookup('config', application_id, 'services.moodle.performance.php.upload_max_filesize') }}"
MOODLE_PHP_POST_MAX_SIZE: "{{ lookup('config', application_id, 'services.moodle.performance.php.post_max_size') }}"
MOODLE_PHP_MAX_EXECUTION_TIME: "300"
roles/web-app-odoo/templates/odoo.conf.j2
+2 -2
View File
@@ -28,8 +28,8 @@ gevent_port = {{ container_websocket_port }}
proxy_mode = True
workers = 4
max_cron_threads = 2
limit_memory_hard = 2684354560
limit_memory_soft = 2147483648
limit_memory_hard = {{ (lookup('service_memory_mb', application_id, 'odoo', 0.625) | int) * 1048576 }}
limit_memory_soft = {{ (lookup('service_memory_mb', application_id, 'odoo', 0.5) | int) * 1048576 }}
limit_time_cpu = 600
limit_time_real = 1200
limit_time_real_cron = 600
roles/web-app-shopware/files/shell/init.sh
+5 -4
View File
@@ -11,6 +11,7 @@ APP_ROOT="/var/www/html"
MARKER="$APP_ROOT/.infinito/installed"
LOG_PREFIX="[INIT]"
PHP_BIN="php"
PHP_MEMORY_LIMIT="${INIT_PHP_MEMORY_LIMIT:?INIT_PHP_MEMORY_LIMIT must be set}"
log() { printf "%s %s\n" "$LOG_PREFIX" "$1"; }
fail() { printf "%s [ERROR] %s\n" "$LOG_PREFIX" "$1" >&2; exit 1; }
@@ -104,18 +105,18 @@ fi
if [ "$FIRST_INSTALL" -eq 1 ]; then
log "Performing first-time Shopware installation..."
$PHP_BIN -d memory_limit=1024M bin/console system:install --basic-setup --create-database
$PHP_BIN -d memory_limit="$PHP_MEMORY_LIMIT" bin/console system:install --basic-setup --create-database
log "Locking the 'admin' account that --basic-setup creates with the published default password..."
DEFAULT_ADMIN_PASSWORD="$($PHP_BIN -r 'echo bin2hex(random_bytes(32));')"
$PHP_BIN -d memory_limit=1024M bin/console user:change-password admin \
$PHP_BIN -d memory_limit="$PHP_MEMORY_LIMIT" bin/console user:change-password admin \
--password="$DEFAULT_ADMIN_PASSWORD"
mkdir -p "$(dirname "$MARKER")"
: > "$MARKER"
fi
log "Running database migrations..."
$PHP_BIN -d memory_limit=1024M bin/console database:migrate --all
$PHP_BIN -d memory_limit=1024M bin/console database:migrate-destructive --all
$PHP_BIN -d memory_limit="$PHP_MEMORY_LIMIT" bin/console database:migrate --all
$PHP_BIN -d memory_limit="$PHP_MEMORY_LIMIT" bin/console database:migrate-destructive --all
log "Rebuilding caches and assets..."
$PHP_BIN bin/console cache:clear
roles/web-app-shopware/meta/services.yml
-1
View File
@@ -98,7 +98,6 @@ worker:
- async
- low_priority
- --time-limit=300
- --memory-limit=512M
cpus: 1.0
mem_reservation: 1g
mem_limit: 2g
roles/web-app-shopware/templates/compose.yml.j2
+4 -1
View File
@@ -12,6 +12,8 @@ x-environment: &shopware
{{ lookup('compose_only', 'container_name', SHOPWARE_INIT_CONTAINER) }}
entrypoint: [ "sh", "{{ SHOPWARE_INIT_DOCKER }}" ]
user: "0:0"
environment:
INIT_PHP_MEMORY_LIMIT: "{{ lookup('service_memory_mb', application_id, service_name, 0.5) }}M"
{{ lookup('container_volumes', service_name) | indent(4) }}
{{ lookup('container_networks') | indent(4) }}
@@ -53,6 +55,7 @@ x-environment: &shopware
<<: *shopware
{{ lookup('compose_only', 'container_name', SHOPWARE_WORKER_CONTAINER) }}
entrypoint: {{ SHOPWARE_WORKER_ENTRYPOINT }}
command: ["--memory-limit={{ lookup('service_memory_mb', application_id, service_name, 0.25) }}M"]
{{ lookup('depends_on', {'init': 'service_completed_successfully'}) }}
{{ lookup('container_healthcheck', service_name) | indent(4) }}
{{ lookup('container_volumes', service_name) | indent(4) }}
@@ -82,7 +85,7 @@ x-environment: &shopware
- discovery.type=single-node
- plugins.security.disabled=true
- bootstrap.memory_lock=true
- OPENSEARCH_JAVA_OPTS=-Xms{{ SHOPWARE_OPENSEARCH_MEM_RESERVATION }} -Xmx{{ SHOPWARE_OPENSEARCH_MEM_RESERVATION }}
- OPENSEARCH_JAVA_OPTS=-Xms{{ SHOPWARE_OPENSEARCH_HEAP_MB }}m -Xmx{{ SHOPWARE_OPENSEARCH_HEAP_MB }}m
ulimits:
memlock: { soft: -1, hard: -1 }
{{ lookup('depends_on', {'init': 'service_completed_successfully'}) }}
roles/web-app-shopware/vars/main.yml
+1 -1
View File
@@ -44,7 +44,7 @@ SHOPWARE_REDIS_URL: "{{ lookup('engine', 'redis', application_id, 'url') }}"
SHOPWARE_OPENSEARCH_PORT: "{{ lookup('config', application_id, 'services.opensearch.ports.internal.http') }}"
SHOPWARE_OPENSEARCH_ENABLED: "{{ lookup('config', application_id, 'services.opensearch.enabled') }}"
SHOPWARE_OPENSEARCH_CONTAINER: "{{ lookup('config', application_id, 'services.opensearch.name') }}"
SHOPWARE_OPENSEARCH_MEM_RESERVATION: "{{ lookup('config', application_id, 'services.opensearch.mem_reservation') }}"
SHOPWARE_OPENSEARCH_HEAP_MB: "{{ lookup('service_memory_mb', application_id, 'opensearch', 0.5) }}"
SHOPWARE_OPENSEARCH_INITIAL_ADMIN_PASSWORD: "{{ lookup('config', application_id, 'secrets.credentials.opensearch_initial_admin_password') }}"
swarm_stack_fatal_grace: 600
roles/web-app-snipe-it/tasks/01_install.yml
+1 -1
View File
@@ -41,7 +41,7 @@
-e APP_KEY="{{ SNIPE_IT_APP_KEY }}"
-e XDG_CONFIG_HOME=/tmp
{{ SNIPE_IT_CONTAINER_ADDRESS }}
php -d memory_limit=512M artisan db:seed --force
php -d memory_limit={{ lookup('service_memory_mb', application_id, SNIPE_IT_SERVICE, 0.25) }}M artisan db:seed --force
register: snipeit_seed
changed_when: "'Seeding:' in snipeit_seed.stdout"
failed_when: snipeit_seed.rc != 0
roles/web-app-suitecrm/files/Dockerfile
+3 -2
View File
@@ -4,6 +4,7 @@ ARG SUITECRM_BASE_VERSION
FROM ${SUITECRM_BASE_IMAGE}:${SUITECRM_BASE_VERSION} AS base
ARG SUITECRM_APP_DIR
ARG SUITECRM_PHP_MEMORY_LIMIT
RUN rm -rf /var/lib/apt/lists/* \
&& apt-get -o Acquire::Retries=3 update && apt-get -o Acquire::Retries=3 install -y \
@@ -31,11 +32,11 @@ RUN rm -rf /var/lib/apt/lists/* \
ldap \
&& rm -rf /var/lib/apt/lists/*
RUN { \
RUN : "${SUITECRM_PHP_MEMORY_LIMIT:?}" && { \
echo 'file_uploads = On'; \
echo 'upload_max_filesize = 32M'; \
echo 'post_max_size = 32M'; \
echo 'memory_limit = 512M'; \
echo "memory_limit = ${SUITECRM_PHP_MEMORY_LIMIT}"; \
} > /usr/local/etc/php/conf.d/suitecrm-upload.ini
RUN a2enmod rewrite headers
roles/web-app-suitecrm/templates/compose.yml.j2
+1
View File
@@ -10,6 +10,7 @@
SUITECRM_ENTRYPOINT_SCRIPT_DOCKER: "{{ SUITECRM_ENTRYPOINT_SCRIPT_DOCKER }}"
SUITECRM_LDAP_EXTENSION_DIR: "{{ SUITECRM_LDAP_EXTENSION_DIR }}"
SUITECRM_APP_DIR: "{{ SUITECRM_APP_DIR }}"
SUITECRM_PHP_MEMORY_LIMIT: "{{ SUITECRM_PHP_MEMORY_LIMIT }}"
SUITECRM_SEED_DIR: "{{ SUITECRM_SEED_DIR }}"
SUITECRM_WEB_USER: "{{ SUITECRM_WEB_USER }}"
SUITECRM_WEB_GROUP: "{{ SUITECRM_WEB_GROUP }}"
roles/web-app-suitecrm/vars/main.yml
+1
View File
@@ -39,3 +39,4 @@ SUITECRM_ENTRYPOINT_SCRIPT_HOST_REL: "volumes/{{ SUITECRM_ENTRYPOINT_SCRIPT_FILE
SUITECRM_ENTRYPOINT_SCRIPT_DOCKER: "{{ [ '/usr/local/bin/', SUITECRM_ENTRYPOINT_SCRIPT_FILE ] | path_join }}"
SUITECRM_MAX_OLD_SPACE_SIZE: "{{ lookup('node_max_old_space_size', application_id, entity_name) }}"
SUITECRM_PHP_MEMORY_LIMIT: "{{ lookup('service_memory_mb', application_id, entity_name, 0.25) }}M"
roles/web-app-zammad/templates/compose.yml.j2
+1 -1
View File
@@ -61,7 +61,7 @@
{{ lookup('container_image', 'svc-db-memcached', 'memcached') }}
{{ lookup('compose_only', 'container_name', ZAMMAD_MEMCACHED_CONTAINER) }}
{% include 'roles/sys-svc-container/templates/base.yml.j2' %}
command: ["memcached", "-m", "256"]
command: ["memcached", "-m", "{{ lookup('service_memory_mb', application_id, service_name, 0.125) }}"]
{{ lookup('container_extra_hosts') | indent(4) }}
{{ lookup('container_networks') | indent(4) }}
{% endif %}
roles/web-app-zammad/templates/env.j2
+1 -1
View File
@@ -23,7 +23,7 @@ ELASTICSEARCH_NAMESPACE=zammad
ELASTICSEARCH_REINDEX=true
discovery.type=single-node
xpack.security.enabled=false
ES_JAVA_OPTS=-Xms512m -Xmx512m
ES_JAVA_OPTS=-Xms{{ ZAMMAD_SEARCH_HEAP_MB }}m -Xmx{{ ZAMMAD_SEARCH_HEAP_MB }}m
RAILS_ENV=production
RAILS_LOG_TO_STDOUT=true
roles/web-app-zammad/vars/main.yml
+1
View File
@@ -22,6 +22,7 @@ ZAMMAD_MEMCACHED_CONTAINER: "{{ lookup('engine', 'memcached', application_id, 'c
ZAMMAD_RAILS_PORT: "{{ lookup('config', application_id, 'services.' ~ ZAMMAD_RAILS_SERVICE ~ '.ports.internal.http') }}"
ZAMMAD_WEBSOCKET_PORT: "{{ lookup('config', application_id, 'services.' ~ ZAMMAD_WEBSOCKET_SERVICE ~ '.ports.internal.http') }}"
ZAMMAD_SEARCH_PORT: "{{ lookup('config', application_id, 'services.' ~ ZAMMAD_SEARCH_SERVICE ~ '.ports.internal.http') }}"
ZAMMAD_SEARCH_HEAP_MB: "{{ lookup('service_memory_mb', application_id, ZAMMAD_SEARCH_SERVICE, 0.5) }}"
ZAMMAD_DB_POOL: 50
tests/unit/python/plugins/filter/test_memory_filters.py
-234
View File
@@ -1,234 +0,0 @@
import importlib
import unittest
from unittest.mock import patch
memory_filters = importlib.import_module("plugins.filter.memory_filters")
class TestMemoryFilters(unittest.TestCase):
def setUp(self):
self.apps = {"whatever": True}
self.app_id = "web-app-confluence" # entity_name will be mocked
# -----------------------------
# Helpers
# -----------------------------
def _with_conf(self, mem_limit: str, mem_res: str):
"""
Patch get/entity_name so that mem_limit and mem_reservation
can be controlled in tests.
"""
patches = [
patch(
"plugins.filter.memory_filters.entity_name",
return_value="confluence",
),
patch(
"plugins.filter.memory_filters.get",
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
mem_limit
if key.endswith(".mem_limit")
else mem_res
if key.endswith(".mem_reservation")
else None
),
),
]
mocks = [p.start() for p in patches]
self.addCleanup(lambda: [p.stop() for p in patches])
return mocks
# -----------------------------
# Tests: jvm_max_mb / jvm_min_mb sizing
# -----------------------------
def test_sizing_8g_limit_6g_reservation(self):
self._with_conf("8g", "6g")
xmx = memory_filters.jvm_max_mb(self.apps, self.app_id)
xms = memory_filters.jvm_min_mb(self.apps, self.app_id)
self.assertEqual(xmx, 5734)
self.assertEqual(xms, 2867)
def test_sizing_6g_limit_4g_reservation(self):
self._with_conf("6g", "4g")
xmx = memory_filters.jvm_max_mb(self.apps, self.app_id)
xms = memory_filters.jvm_min_mb(self.apps, self.app_id)
self.assertEqual(xmx, 4300)
self.assertEqual(xms, 2150)
def test_sizing_16g_limit_12g_reservation_cap_12288(self):
self._with_conf("16g", "12g")
xmx = memory_filters.jvm_max_mb(self.apps, self.app_id)
xms = memory_filters.jvm_min_mb(self.apps, self.app_id)
self.assertEqual(xmx, 11468)
self.assertEqual(xms, 5734)
def test_floor_small_limit_results_in_min_1024(self):
self._with_conf("1g", "512m")
xmx = memory_filters.jvm_max_mb(self.apps, self.app_id)
self.assertEqual(xmx, 1024)
def test_floor_small_reservation_results_in_min_512(self):
self._with_conf("4g", "128m")
xms = memory_filters.jvm_min_mb(self.apps, self.app_id)
self.assertEqual(xms, 512)
# -----------------------------
# Tests: JVM failure cases / validation
# -----------------------------
def test_invalid_unit_raises(self):
with (
patch(
"plugins.filter.memory_filters.entity_name",
return_value="confluence",
),
patch(
"plugins.filter.memory_filters.get",
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
"8Q" if key.endswith(".mem_limit") else "4g"
),
),
self.assertRaises(memory_filters.AnsibleFilterError),
):
memory_filters.jvm_max_mb(self.apps, self.app_id)
def test_zero_limit_raises(self):
with (
patch(
"plugins.filter.memory_filters.entity_name",
return_value="confluence",
),
patch(
"plugins.filter.memory_filters.get",
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
"0" if key.endswith(".mem_limit") else "4g"
),
),
self.assertRaises(memory_filters.AnsibleFilterError),
):
memory_filters.jvm_max_mb(self.apps, self.app_id)
def test_zero_reservation_raises(self):
with (
patch(
"plugins.filter.memory_filters.entity_name",
return_value="confluence",
),
patch(
"plugins.filter.memory_filters.get",
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
"8g" if key.endswith(".mem_limit") else "0"
),
),
self.assertRaises(memory_filters.AnsibleFilterError),
):
memory_filters.jvm_min_mb(self.apps, self.app_id)
def test_entity_name_is_derived_not_passed(self):
"""
Ensure entity_name() is called internally and the app_id is not
passed around manually from the template.
"""
with (
patch(
"plugins.filter.memory_filters.entity_name",
return_value="confluence",
) as mock_entity,
patch(
"plugins.filter.memory_filters.get",
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
"8g" if key.endswith(".mem_limit") else "6g"
),
),
):
xmx = memory_filters.jvm_max_mb(self.apps, self.app_id)
xms = memory_filters.jvm_min_mb(self.apps, self.app_id)
self.assertGreater(xmx, 0)
self.assertGreater(xms, 0)
self.assertEqual(mock_entity.call_count, 3)
for call in mock_entity.call_args_list:
self.assertEqual(call.args[0], self.app_id)
# -----------------------------
# Tests: redis_maxmemory_mb
# -----------------------------
def test_redis_maxmemory_default_factor_uses_80_percent_of_limit(self):
self._with_conf("1g", "512m")
maxmem = memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
self.assertEqual(maxmem, 819)
def test_redis_maxmemory_custom_factor_and_min_mb(self):
self._with_conf("1g", "512m")
maxmem = memory_filters.redis_maxmemory_mb(
self.apps,
self.app_id,
factor=0.5,
min_mb=128,
)
self.assertEqual(maxmem, 512)
def test_redis_maxmemory_honors_minimum_floor(self):
self._with_conf("32m", "16m")
maxmem = memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
self.assertEqual(maxmem, 64)
def test_redis_maxmemory_zero_limit_raises(self):
self._with_conf("0", "512m")
with self.assertRaises(memory_filters.AnsibleFilterError):
memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
def test_redis_maxmemory_invalid_unit_raises(self):
self._with_conf("8Q", "512m")
with self.assertRaises(memory_filters.AnsibleFilterError):
memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
def test_redis_maxmemory_does_not_call_entity_name(self):
"""
Ensure redis_maxmemory_mb does NOT rely on entity name resolution
(it should always use the hard-coded 'redis' service name).
"""
patches = [
patch("plugins.filter.memory_filters.entity_name"),
patch(
"plugins.filter.memory_filters.get",
side_effect=lambda apps, app_id, key, required=True, **kwargs: (
"4g" if key.endswith(".mem_limit") else "2g"
),
),
]
mocks = [p.start() for p in patches]
self.addCleanup(lambda: [p.stop() for p in patches])
entity_mock = mocks[0]
maxmem = memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
self.assertEqual(maxmem, 3276)
entity_mock.assert_not_called()
def test_redis_maxmemory_uses_default_when_mem_limit_missing(self):
"""
When services.redis.mem_limit is not configured, the filter
should fall back to its internal default (256m).
"""
def fake_get(apps, app_id, key, required=True, **kwargs):
if key.endswith(".mem_limit"):
return kwargs.get("default")
return None
with (
patch(
"plugins.filter.memory_filters.get",
side_effect=fake_get,
),
patch(
"plugins.filter.memory_filters.entity_name",
return_value="confluence",
),
):
maxmem = memory_filters.redis_maxmemory_mb(self.apps, self.app_id)
self.assertEqual(maxmem, 204)
if __name__ == "__main__":
unittest.main()
tests/unit/python/plugins/lookup/test_service_memory_mb.py
+101
View File
@@ -0,0 +1,101 @@
from __future__ import annotations
import unittest
from unittest.mock import MagicMock, patch
from ansible.errors import AnsibleError
from plugins.lookup.service_memory_mb import LookupModule, docker_bytes, share_mb
_APP = "web-app-demo"
class TestDockerBytes(unittest.TestCase):
def test_suffixes_are_binary(self) -> None:
self.assertEqual(docker_bytes("512m"), 512 * 1024**2)
self.assertEqual(docker_bytes("2g"), 2 * 1024**3)
self.assertEqual(docker_bytes("1.5g"), 1536 * 1024**2)
self.assertEqual(docker_bytes("64K"), 64 * 1024)
self.assertEqual(docker_bytes("1gb"), 1024**3)
def test_a_bare_number_is_bytes(self) -> None:
self.assertEqual(docker_bytes(1048576), 1024**2)
self.assertEqual(docker_bytes("2048"), 2048)
def test_an_unreadable_size_raises(self) -> None:
for size in ("", "lots", "2x", "-1g", None):
with self.subTest(size=size), self.assertRaises(AnsibleError):
docker_bytes(size)
class TestShareMb(unittest.TestCase):
def test_share_of_the_limit_in_whole_mebibytes(self) -> None:
cases = (
("2g", 0.5, 1024),
("4g", 0.5, 2048),
("8g", 0.7, 5734),
("8g", 0.35, 2867),
("6g", 0.7, 4300),
("512m", 0.8, 409),
("256m", 0.75, 192),
("1.5g", 0.5, 768),
("1g", 1, 1024),
)
for mem_limit, share, expected in cases:
with self.subTest(mem_limit=mem_limit, share=share):
self.assertEqual(share_mb(mem_limit, share), expected)
def test_a_share_outside_the_limit_raises(self) -> None:
for share in (0, -0.5, 1.01, "half", None):
with self.subTest(share=share), self.assertRaises(AnsibleError):
share_mb("2g", share)
def test_a_result_below_one_mebibyte_raises(self) -> None:
with self.assertRaises(AnsibleError):
share_mb("1m", 0.5)
class TestLookupResolvesTheContainerLimit(unittest.TestCase):
def _run(self, services: dict, variables: dict, terms: list) -> int:
templar = MagicMock()
templar.available_variables = variables
templar.template.side_effect = lambda value: {"{{ HOST }}": "1.5g"}.get(
value, value
)
applications = MagicMock()
applications.run.return_value = [{_APP: {"services": services}}]
lookup = LookupModule(loader=None, templar=templar)
with patch(
"plugins.lookup.service_memory_mb.lookup_loader.get",
return_value=applications,
):
return lookup.run(terms, variables=variables)[0]
def test_the_named_service_wins(self) -> None:
services = {"search": {"mem_limit": "2g"}, "demo": {"mem_limit": "8g"}}
self.assertEqual(self._run(services, {}, [_APP, "search", 0.5]), 1024)
def test_the_primary_entity_backs_an_undeclared_service(self) -> None:
services = {"search": {}, "demo": {"mem_limit": "8g"}}
self.assertEqual(self._run(services, {}, [_APP, "search", 0.5]), 4096)
def test_the_host_default_backs_an_undeclared_role(self) -> None:
variables = {"RESOURCE_MEM_LIMIT": "{{ HOST }}"}
self.assertEqual(
self._run({"search": {}}, variables, [_APP, "search", 0.5]), 768
)
def test_no_limit_anywhere_raises(self) -> None:
with self.assertRaises(AnsibleError):
self._run({"search": {}}, {}, [_APP, "search", 0.5])
def test_the_term_count_is_checked(self) -> None:
with self.assertRaises(AnsibleError):
self._run({"search": {"mem_limit": "2g"}}, {}, [_APP, "search"])
if __name__ == "__main__":
unittest.main()