fix(sys-svc-rdbms): let a dedicated mariadb realign its own credentials

An engine of its own keeps its passwords in its data directory, written once
when that directory was initialised. When the deploy rotates the credential the
new value reaches the container's environment and never the engine, so the
health check authenticates with a password nothing has ever accepted. The
application user and root come from the same value, so nothing is left that
could log in and change it either.

mariadbd reads --init-file on every start, before it opens the socket, and runs
the statements with full privileges. That is the one point where a password can
be reset without knowing the previous one.

The SQL is rendered next to the env file that already holds the same secret in
plaintext, so it adds no new exposure. It reaches the container as a config
rather than a bind mount because a swarm task may be scheduled on a node the
rendered path does not exist on; the config name carries a hash of the file
content, so a rotated password produces a new name and swarm replaces the
immutable old one.

The guard is mariadb-and-not-shared, which matches the `local` guard the service
template already renders on, so the block and the top-level entry appear
together or not at all.

3831c8f804 fixed the other half of this failure: matomo, moodle and nextcloud
keep the same password in a config file on their data volume. Same rotation,
opposite end of the connection.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

plugins/lookup/compose_volumes.py

@@ -28,6 +28,7 @@ from utils.roles.applications.mounts import (
     normalize_volumes_meta,
 )
 from utils.roles.applications.services.database import (
+    REALIGN_CONFIG_KEY,
     get_database_service_config,
     resolve_database_service_key,
 )
@@ -298,6 +299,44 @@ def compose_volumes(
 
 
 class LookupModule(LookupBase):
+    def _with_database_realign(
+        self,
+        applications: dict[str, Any],
+        application_id: str,
+        vars_: dict[str, Any],
+        extra_configs: dict[str, Any] | None,
+    ) -> dict[str, Any] | None:
+        """Add the realignment statements an app's own mariadb reads at start.
+
+        Args:
+            applications: the merged applications mapping.
+            application_id: the app whose stack is being rendered.
+            vars_: the variables in scope for the lookup.
+            extra_configs: configs a caller already supplied.
+
+        The file lands as a config rather than a bind mount because a swarm task
+        may run on a node the source path does not exist on.
+        """
+        if resolve_database_service_key(applications, application_id) != "mariadb":
+            return extra_configs
+        if get_database_service_config(applications, application_id).get("shared"):
+            return extra_configs
+
+        source = lookup_loader.get(
+            "database", loader=self._loader, templar=getattr(self, "_templar", None)
+        ).run([application_id, "realign_sql"], variables=vars_)[0]
+        if not source:
+            return extra_configs
+
+        entity = get_entity_name(application_id)
+        return {
+            **(extra_configs or {}),
+            REALIGN_CONFIG_KEY: {
+                "name": _config_secret_name(entity, REALIGN_CONFIG_KEY, source),
+                "file": source,
+            },
+        }
+
     def run(
         self,
         terms: list[Any] | None,
@@ -366,7 +405,9 @@ class LookupModule(LookupBase):
             applications,
             application_id,
             extra_volumes=kwargs.get("extra_volumes"),
-            extra_configs=kwargs.get("extra_configs"),
+            extra_configs=self._with_database_realign(
+                applications, application_id, vars_, kwargs.get("extra_configs")
+            ),
             extra_secrets=kwargs.get("extra_secrets"),
             deployment_mode=str(deployment_mode).strip(),
             storage=storage,

plugins/lookup/database.py

@@ -11,6 +11,8 @@ from ansible.plugins.lookup import LookupBase
 from utils.manager.credential_key import OVERRIDE_SECTION
 from utils.roles.applications.config import get
 from utils.roles.applications.services.database import (
+    REALIGN_CONFIG_KEY,
+    REALIGN_TARGET,
     get_database_service_config,
     resolve_database_service_key,
 )
@@ -102,6 +104,9 @@ class LookupModule(LookupBase):
                 "password": "",
                 "port": "",
                 "env": "",
+                "realign_sql": "",
+                "realign_target": REALIGN_TARGET,
+                "realign_config": REALIGN_CONFIG_KEY,
                 "initdb_dir": "",
                 "build_dir": "",
                 "url_jdbc": "",
@@ -190,6 +195,7 @@ class LookupModule(LookupBase):
 
         env_dir = f"{path_instances}{get_entity_name(consumer_id)}/.env/"
         env = f"{env_dir}{dbtype}.env"
+        realign_sql = f"{env_dir}{dbtype}-realign.sql"
         initdb_dir = f"{path_instances}{get_entity_name(consumer_id)}/.initdb.d/"
         build_dir = f"{path_instances}{get_entity_name(consumer_id)}/.postgres-build/"
 
@@ -251,6 +257,9 @@ class LookupModule(LookupBase):
             "password": password,
             "port": port,
             "env": env,
+            "realign_sql": realign_sql,
+            "realign_target": REALIGN_TARGET,
+            "realign_config": REALIGN_CONFIG_KEY,
             "initdb_dir": initdb_dir,
             "build_dir": build_dir,
             "url_jdbc": url_jdbc,

roles/sys-svc-rdbms/tasks/dedicated.yml

@@ -11,6 +11,14 @@
     dest: "{{ _database_env }}"
   notify: compose-up
 
+- name: "📄 Render the credential realignment for '{{ _database_consumer_id }}'"
+  stack_host_template:
+    src: "sql/mariadb-realign.sql.j2"
+    dest: "{{ lookup('database', _database_consumer_id, 'realign_sql') }}"
+    mode: "0640"
+  when: _database_type == 'mariadb'
+  notify: compose-up
+
 - name: "🔧 For '{{ application_id }}/{{ _database_consumer_id }}': Apply postgres setup"
   include_tasks: dedicated_postgres.yml
   when: _database_type == 'postgres'

roles/sys-svc-rdbms/templates/services/mariadb.yml.j2

@@ -10,7 +10,10 @@
     {{ lookup('compose_restart') }}
     env_file:
       - {{ lookup('database', application_id, 'env') }}
-    command: "--transaction-isolation=READ-COMMITTED --binlog-format=ROW"
+    command: "--transaction-isolation=READ-COMMITTED --binlog-format=ROW --init-file={{ lookup('database', application_id, 'realign_target') }}"
+    configs:
+      - source: {{ lookup('database', application_id, 'realign_config') }}
+        target: {{ lookup('database', application_id, 'realign_target') }}
     volumes:
       - database:/var/lib/mysql
     healthcheck:

roles/sys-svc-rdbms/templates/sql/mariadb-realign.sql.j2

@@ -0,0 +1,8 @@
+CREATE USER IF NOT EXISTS '{{ lookup('database', application_id, 'username') }}'@'%'
+  IDENTIFIED BY '{{ lookup('database', application_id, 'password') }}';
+ALTER USER '{{ lookup('database', application_id, 'username') }}'@'%'
+  IDENTIFIED BY '{{ lookup('database', application_id, 'password') }}';
+ALTER USER 'root'@'localhost'
+  IDENTIFIED BY '{{ lookup('database', application_id, 'password') }}';
+GRANT ALL PRIVILEGES ON `{{ lookup('database', application_id, 'name') }}`.*
+  TO '{{ lookup('database', application_id, 'username') }}'@'%';

utils/roles/applications/services/database.py

@@ -7,6 +7,9 @@ if TYPE_CHECKING:
 
 RDBMS_SERVICE_KEYS = ("mariadb", "postgres")
 
+REALIGN_CONFIG_KEY = "database_realign"
+REALIGN_TARGET = "/etc/mysql/realign.sql"
+
 
 def _as_mapping(value: Any) -> dict[str, Any]:
     return value if isinstance(value, dict) else {}