perf(web-app-keycloak): reconcile per-app RBAC over the admin REST API

The per-app RBAC groups and LDAP group mappers were reconciled by four
kcadm shell scripts that started a JVM per call and re-synced every app
on every deploy: 582 s for the mappers plus 324 s for the orphan purge on
echoserver (39 apps).

rbac_reconcile.py now talks to the admin REST API from inside the
Keycloak network namespace (nsenter --net) with the admin credentials
read from /proc/<pid>/environ, so nothing secret reaches argv or output.
It drops the shared ldap-roles mapper, purges root groups of undeployed
apps, ensures each app group and mapper, and syncs a mapper only when the
group is new, the mapper config drifted, or the fingerprint stored on the
group (mapper payload, app rbac and domains, group-logic digest) differs.

The per-app payload no longer carries groups.path.group: Keycloak 26.8.1
does not store that key, so a payload with it never matched the stored
mapper and every deploy re-synced every app.

The admin client lives in keycloak_admin.py; tasks/utils/rest_script.yml
stages it with the script in a 0700 temp dir on KEYCLOAK_HOST_NODE, runs
it and removes the dir in an always block.

Evidence (local compose, slot 104): matrix iteration green in both
variants and passes; update runs show synced=1 in 1.1 s, then
synced=0 unchanged=1 in 0.81 s through the final delivery path, no
staging dir left behind.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

roles/web-app-keycloak/files/python/keycloak_admin.py

@@ -0,0 +1,131 @@
+"""Keycloak admin REST client shared by the scripts beside this module.
+
+Exports:
+    KeycloakError: an HTTP error status, naming method, path and status but never the body.
+    container_credentials: the permanent admin read from ``/proc/<pid>/environ`` of the Keycloak container.
+    Keycloak: client whose ``call`` paths are relative to ``/admin/realms/<realm>/``.
+    ADMIN_ENV_KEYS: container environment keys of the permanent admin user and password.
+    PAGE: page size of every paged listing.
+    TOKEN_MARGIN: seconds before expiry at which the master realm token is renewed.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+ADMIN_ENV_KEYS = (
+    "KEYCLOAK_PERMANENT_ADMIN_USERNAME",
+    "KEYCLOAK_PERMANENT_ADMIN_PASSWORD",
+)
+PAGE = 500
+TOKEN_MARGIN = 30
+
+
+class KeycloakError(Exception):
+    pass
+
+
+def container_credentials(environ: Path) -> tuple[str, str]:
+    """Return the admin user and password from the Keycloak container's environment.
+
+    Args:
+        environ: ``/proc/<pid>/environ`` of the Keycloak container.
+    """
+    with environ.open("rb") as handle:
+        raw = handle.read()
+    env = dict(item.split("=", 1) for item in raw.decode().split("\0") if "=" in item)
+    user_key, password_key = ADMIN_ENV_KEYS
+    return env[user_key], env[password_key]
+
+
+class Keycloak:
+    """Admin REST client that renews its master realm token before it expires.
+
+    Args:
+        url: Keycloak base URL.
+        realm: realm every ``call`` path is relative to.
+        user: master realm administrator.
+        password: that administrator's password.
+    """
+
+    def __init__(self, url: str, realm: str, user: str, password: str) -> None:
+        self.url = url.rstrip("/")
+        self.realm = realm
+        self._form = urllib.parse.urlencode(
+            {
+                "client_id": "admin-cli",
+                "grant_type": "password",
+                "username": user,
+                "password": password,
+            }
+        ).encode()
+        self._lock = threading.Lock()
+        self._token = ""
+        self._expires = 0.0
+
+    def _bearer(self) -> str:
+        with self._lock:
+            if time.monotonic() > self._expires - TOKEN_MARGIN:
+                request = urllib.request.Request(  # noqa: S310 fixed local Keycloak origin
+                    f"{self.url}/realms/master/protocol/openid-connect/token",
+                    data=self._form,
+                )
+                with urllib.request.urlopen(request, timeout=30) as response:  # noqa: S310 fixed local Keycloak origin
+                    body = json.load(response)
+                self._token = body["access_token"]
+                self._expires = time.monotonic() + int(body["expires_in"])
+            return f"Bearer {self._token}"
+
+    def call(
+        self, method: str, path: str, payload: object = None, timeout: int = 60
+    ) -> tuple[object, str]:
+        """Return the decoded body and the id the ``Location`` header names.
+
+        Args:
+            method: HTTP method.
+            path: path below ``/admin/realms/<realm>/``.
+            payload: JSON body, or None for none.
+            timeout: seconds the request may take.
+        """
+        data = b"" if payload is None else json.dumps(payload).encode()
+        request = urllib.request.Request(  # noqa: S310 fixed local Keycloak origin
+            f"{self.url}/admin/realms/{self.realm}/{path}",
+            data=data if method in ("POST", "PUT") else None,
+            headers={
+                "Authorization": self._bearer(),
+                "Content-Type": "application/json",
+            },
+            method=method,
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=timeout) as response:  # noqa: S310 fixed local Keycloak origin
+                raw = response.read()
+                location = response.headers.get("Location") or ""
+        except urllib.error.HTTPError as error:
+            raise KeycloakError(f"{method} {path}: HTTP {error.code}") from None
+        return (json.loads(raw) if raw else None), location.rsplit("/", 1)[-1]
+
+    def children(self, group_id: str) -> list[dict]:
+        """Return every direct subgroup of a group, page by page.
+
+        Args:
+            group_id: the parent group.
+        """
+        found: list[dict] = []
+        while True:
+            query = urllib.parse.urlencode(
+                {"first": len(found), "max": PAGE, "briefRepresentation": "true"}
+            )
+            page, _ = self.call("GET", f"groups/{group_id}/children?{query}")
+            found.extend(page)
+            if len(page) < PAGE:
+                return found

roles/web-app-keycloak/files/python/rbac_reconcile.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python3
+"""Reconcile Keycloak's per-application RBAC groups and LDAP group mappers over the admin REST API.
+
+Env:
+    KC_URL: Keycloak base URL reachable from the network namespace the script runs in.
+    KC_PID: host PID of the Keycloak container, whose environment holds the admin credentials.
+    KC_REALM: realm holding the RBAC groups.
+    LDAP_COMPONENT_ID: LDAP user storage component id, parent of the group mappers.
+    RBAC_ROOT_PATH: path of the RBAC root group, e.g. ``/roles``.
+    DEPLOYED_APP_IDS: space-separated application ids whose groups stay under the root.
+    PAYLOADS_FILE: one ``<app_id> <mapper_name> <base64 mapper JSON> <fingerprint>`` line per app.
+    WORKERS: apps reconciled in parallel.
+    SYNC_ATTEMPTS: mapper sync attempts per app.
+    SYNC_DELAY: seconds between two sync attempts.
+    SYNC_TIMEOUT: seconds one sync attempt may take.
+
+Output: one ``<step>-failed <app_id>`` line per failed app, then one ``summary`` line.
+Exit: 0 every app reconciled, 2 only mapper syncs failed, 1 otherwise.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import sys
+import time
+import urllib.parse
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from pathlib import Path
+
+from keycloak_admin import Keycloak, KeycloakError, container_credentials
+
+SHARED_MAPPER = "ldap-roles"
+MAPPER_TYPE = "org.keycloak.storage.ldap.mappers.LDAPStorageMapper"
+FINGERPRINT = "infinito.rbac.fingerprint"
+
+
+class StepError(Exception):
+    def __init__(self, step: str, cause: Exception) -> None:
+        super().__init__(f"{step}: {cause}")
+        self.step = step
+
+
+@dataclass
+class App:
+    app_id: str
+    mapper_name: str
+    payload: str
+    fingerprint: str
+
+
+def read_payloads(path: str) -> list[App]:
+    """Return the apps of a payloads file.
+
+    Args:
+        path: file with one ``<app_id> <mapper_name> <base64 JSON> <fingerprint>`` line per app.
+    """
+    with Path(path).open(encoding="utf-8") as handle:
+        lines = handle.read().splitlines()
+    apps = []
+    for line in lines:
+        if line.strip():
+            app_id, mapper_name, encoded, fingerprint = line.split()
+            apps.append(
+                App(
+                    app_id, mapper_name, base64.b64decode(encoded).decode(), fingerprint
+                )
+            )
+    return apps
+
+
+def ldap_mappers(kc: Keycloak, ldap_id: str) -> dict[str, dict]:
+    """Return the LDAP group mappers by name.
+
+    Args:
+        kc: client.
+        ldap_id: LDAP user storage component id.
+    """
+    query = urllib.parse.urlencode({"parent": ldap_id, "type": MAPPER_TYPE})
+    found, _ = kc.call("GET", f"components?{query}")
+    return {mapper["name"]: mapper for mapper in found}
+
+
+def drop_shared_mapper(kc: Keycloak, mappers: dict[str, dict]) -> list[str]:
+    """Delete the shared mapper whose flat groups would shadow the per-app ones.
+
+    Args:
+        kc: client.
+        mappers: LDAP group mappers by name.
+
+    Returns:
+        One ``drop-failed`` line per mapper that could not be deleted.
+    """
+    shared = mappers.pop(SHARED_MAPPER, None)
+    if shared is None:
+        return []
+    try:
+        kc.call("DELETE", f"components/{shared['id']}")
+    except (KeycloakError, OSError) as error:
+        return [f"drop-failed {SHARED_MAPPER}: {error}"]
+    return []
+
+
+def purge_orphans(
+    kc: Keycloak, root_id: str, keep: set[str], workers: int
+) -> tuple[dict[str, str], list[str]]:
+    """Delete root subgroups that belong to no deployed app.
+
+    Args:
+        kc: client.
+        root_id: RBAC root group id.
+        keep: application ids whose groups stay.
+        workers: parallel deletes.
+
+    Returns:
+        The kept subgroup ids by name, and one ``purge-failed`` line per failed delete.
+    """
+    kept, orphans = {}, []
+    for group in kc.children(root_id):
+        if group["name"] in keep:
+            kept[group["name"]] = group["id"]
+        else:
+            orphans.append(group)
+
+    def delete(group: dict) -> str:
+        try:
+            kc.call("DELETE", f"groups/{group['id']}")
+        except (KeycloakError, OSError) as error:
+            return f"purge-failed {group['name']}: {error}"
+        return ""
+
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        failures = [line for line in pool.map(delete, orphans) if line]
+    return kept, failures
+
+
+def config_matches(current: dict, desired: dict) -> bool:
+    """True when every config key the payload sets already has its value.
+
+    Args:
+        current: mapper as Keycloak returns it.
+        desired: mapper payload.
+    """
+    have = current.get("config") or {}
+    return all(have.get(key) == value for key, value in desired["config"].items())
+
+
+class Reconciler:
+    """Bring one app's group, mapper and LDAP group import up to date.
+
+    Args:
+        kc: client.
+        ldap_id: LDAP user storage component id.
+        root_id: RBAC root group id.
+        groups: existing app group ids by name.
+        mappers: existing LDAP group mappers by name.
+        sync_attempts: mapper sync attempts.
+        sync_delay: seconds between two attempts.
+        sync_timeout: seconds one attempt may take.
+    """
+
+    def __init__(
+        self,
+        kc: Keycloak,
+        ldap_id: str,
+        root_id: str,
+        groups: dict[str, str],
+        mappers: dict[str, dict],
+        sync_attempts: int,
+        sync_delay: float,
+        sync_timeout: int,
+    ) -> None:
+        self.kc = kc
+        self.ldap_id = ldap_id
+        self.root_id = root_id
+        self.groups = groups
+        self.mappers = mappers
+        self.sync_attempts = sync_attempts
+        self.sync_delay = sync_delay
+        self.sync_timeout = sync_timeout
+
+    def _step(self, step: str, action, *args):
+        try:
+            return action(*args)
+        except (KeycloakError, OSError, ValueError, KeyError) as error:
+            raise StepError(step, error) from None
+
+    def _ensure_group(self, app: App) -> tuple[str, bool]:
+        group_id = self.groups.get(app.app_id)
+        if group_id:
+            return group_id, False
+        _, group_id = self.kc.call(
+            "POST", f"groups/{self.root_id}/children", {"name": app.app_id}
+        )
+        if not group_id:
+            raise KeycloakError("created group carries no id")
+        return group_id, True
+
+    def _upsert_mapper(self, app: App) -> tuple[str, bool]:
+        desired = json.loads(app.payload)
+        current = self.mappers.get(app.mapper_name)
+        if current is None:
+            _, mapper_id = self.kc.call("POST", "components", desired)
+            if not mapper_id:
+                raise KeycloakError("created mapper carries no id")
+            return mapper_id, True
+        if config_matches(current, desired):
+            return current["id"], False
+        self.kc.call(
+            "PUT", f"components/{current['id']}", {**desired, "id": current["id"]}
+        )
+        return current["id"], True
+
+    def _sync(self, mapper_id: str) -> None:
+        path = f"user-storage/{self.ldap_id}/mappers/{mapper_id}/sync?direction=fedToKeycloak"
+        for attempt in range(1, self.sync_attempts + 1):
+            try:
+                self.kc.call("POST", path, timeout=self.sync_timeout)
+            except (KeycloakError, OSError):
+                if attempt == self.sync_attempts:
+                    raise
+                time.sleep(self.sync_delay)
+            else:
+                return
+
+    def _record(self, group_id: str, group: dict, fingerprint: str) -> None:
+        attributes = {**(group.get("attributes") or {}), FINGERPRINT: [fingerprint]}
+        self.kc.call("PUT", f"groups/{group_id}", {**group, "attributes": attributes})
+
+    def run(self, app: App) -> str:
+        """Return ``synced`` or ``unchanged`` for an app.
+
+        Args:
+            app: the app to reconcile.
+        """
+        group_id, created = self._step("ensure", self._ensure_group, app)
+        mapper_id, changed = self._step("upsert", self._upsert_mapper, app)
+        group, _ = self._step("read", self.kc.call, "GET", f"groups/{group_id}")
+        stored = ((group.get("attributes") or {}).get(FINGERPRINT) or [""])[0]
+        if not (created or changed or stored != app.fingerprint):
+            return "unchanged"
+        self._step("sync", self._sync, mapper_id)
+        self._step("record", self._record, group_id, group, app.fingerprint)
+        return "synced"
+
+
+def main() -> int:
+    env = os.environ
+    workers = int(env["WORKERS"])
+    try:
+        kc = Keycloak(
+            env["KC_URL"],
+            env["KC_REALM"],
+            *container_credentials(Path("/proc") / env["KC_PID"] / "environ"),
+        )
+        mappers = ldap_mappers(kc, env["LDAP_COMPONENT_ID"])
+        warnings = drop_shared_mapper(kc, mappers)
+        root, _ = kc.call(
+            "GET",
+            f"group-by-path/{urllib.parse.quote(env['RBAC_ROOT_PATH'].strip('/'))}",
+        )
+        groups, purge_failures = purge_orphans(
+            kc, root["id"], set(env["DEPLOYED_APP_IDS"].split()), workers
+        )
+    except (KeycloakError, OSError, KeyError, ValueError) as error:
+        print(f"listing-failed {error}")
+        return 1
+    for line in warnings + purge_failures:
+        print(line, file=sys.stderr)
+
+    reconciler = Reconciler(
+        kc,
+        env["LDAP_COMPONENT_ID"],
+        root["id"],
+        groups,
+        mappers,
+        int(env["SYNC_ATTEMPTS"]),
+        float(env["SYNC_DELAY"]),
+        int(env["SYNC_TIMEOUT"]),
+    )
+
+    def run(app: App) -> tuple[App, str]:
+        try:
+            return app, reconciler.run(app)
+        except StepError as error:
+            print(f"{app.app_id}: {error}", file=sys.stderr)
+            return app, f"{error.step}-failed"
+
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        results = list(pool.map(run, read_payloads(env["PAYLOADS_FILE"])))
+
+    failed = [(app, outcome) for app, outcome in results if outcome.endswith("-failed")]
+    for app, outcome in failed:
+        print(f"{outcome} {app.app_id}")
+    counts = {
+        key: sum(1 for _, o in results if o == key) for key in ("synced", "unchanged")
+    }
+    print(
+        f"summary synced={counts['synced']} unchanged={counts['unchanged']} "
+        f"failed={len(failed)} purged-failures={len(purge_failures)}"
+    )
+    if any(outcome != "sync-failed" for _, outcome in failed):
+        return 1
+    return 2 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

roles/web-app-keycloak/files/shell/per_app_mapper/delete_default_ldap_roles_mapper.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-# Delete the shared default 'ldap-roles' group-ldap-mapper that
-# update/05_ldap/main.yml maintains for the assert/01_ldap.yml contract.
-# The per-application mappers (ldap-roles-<app>) replace it; leaving
-# the default mapper in place would re-import every LDAP groupOfNames
-# flat under /roles on every sync and shadow the per-app structure.
-#
-# Required env:
-#   KC_CONTAINER         keycloak container name
-#   KC_REALM             realm name
-#   LDAP_COMPONENT_ID    Keycloak LDAP component id
-set -o pipefail
-: "${KC_CONTAINER:?KC_CONTAINER is required}"
-: "${KC_REALM:?KC_REALM is required}"
-: "${LDAP_COMPONENT_ID:?LDAP_COMPONENT_ID is required}"
-
-container exec -i "$KC_CONTAINER" /opt/keycloak/bin/kcadm.sh get components \
-  -r "$KC_REALM" \
-  -q "parentId=$LDAP_COMPONENT_ID" \
-  -q providerId=group-ldap-mapper \
-  --fields id,name --format csv --noquotes \
-  | tr -d '\r' \
-  | awk -F',' '$2=="ldap-roles"{print $1}' \
-  > /tmp/kc_ldap_roles_ids.txt
-
-while IFS= read -r m_id; do
-  [ -z "$m_id" ] && continue
-  container exec -i "$KC_CONTAINER" /opt/keycloak/bin/kcadm.sh delete \
-    "components/$m_id" -r "$KC_REALM" </dev/null || true  # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
-done < /tmp/kc_ldap_roles_ids.txt

roles/web-app-keycloak/files/shell/per_app_mapper/purge/root_orphans.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-# Sweep stale flat / orphan Keycloak groups directly under the RBAC
-# root (e.g. /roles/<app>-<role> or /roles/<role-name> collisions
-# from earlier mapper configurations). Children whose name is one of
-# the deployed application IDs are kept; everything else is deleted
-# so the next per-app mapper sync rebuilds a clean tree.
-#
-# Required env:
-#   KC_CONTAINER         keycloak container name
-#   KC_REALM             realm name
-#   KC_RBAC_ROOT_PATH    e.g. /roles
-#   DEPLOYED_APP_IDS     space-separated list of application IDs
-#   KC_PURGE_WORKERS     parallel kcadm deletes
-set -o pipefail
-: "${KC_CONTAINER:?KC_CONTAINER is required}"
-: "${KC_REALM:?KC_REALM is required}"
-: "${KC_RBAC_ROOT_PATH:?KC_RBAC_ROOT_PATH is required}"
-: "${DEPLOYED_APP_IDS:?DEPLOYED_APP_IDS is required}"
-: "${KC_PURGE_WORKERS:?KC_PURGE_WORKERS is required}"
-
-[ -z "$DEPLOYED_APP_IDS" ] && exit 0
-
-container exec -i "$KC_CONTAINER" /opt/keycloak/bin/kcadm.sh get groups \
-  -r "$KC_REALM" \
-  -q max=500 --fields id,path --format csv --noquotes \
-  > /tmp/kc_top_groups.txt
-
-ROOT_ID=$(awk -F',' -v p="$KC_RBAC_ROOT_PATH" \
-  '!/^\[/ && !/Cgroup/ && $2==p {print $1; exit}' \
-  /tmp/kc_top_groups.txt | tr -d '\r')
-[ -z "$ROOT_ID" ] && exit 0
-
-container exec -i "$KC_CONTAINER" /opt/keycloak/bin/kcadm.sh get \
-  "groups/$ROOT_ID/children" -r "$KC_REALM" \
-  -q max=500 --fields id,name --format csv --noquotes \
-  > /tmp/kc_root_children.txt
-
-awk -F',' -v keep="$DEPLOYED_APP_IDS" '
-  BEGIN { n=split(keep,a," "); for(i=1;i<=n;i++) ok[a[i]]=1 }
-  !/^\[/ && !/Cgroup/ && $2 != "" && !ok[$2] { print $1 }
-' /tmp/kc_root_children.txt | tr -d '\r' > /tmp/kc_root_to_delete.txt
-
-xargs -r -P "$KC_PURGE_WORKERS" -I '{}' container exec -i "$KC_CONTAINER" /opt/keycloak/bin/kcadm.sh delete \
-  "groups/{}" -r "$KC_REALM" < /tmp/kc_root_to_delete.txt || true  # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error

roles/web-app-keycloak/files/shell/per_app_mapper/reconcile_apps.sh

@@ -1,147 +0,0 @@
-#!/bin/bash
-# Required env:
-#   KC_CONTAINER          keycloak container name
-#   KC_REALM              realm name
-#   LDAP_COMPONENT_ID     Keycloak LDAP component id (mapper parentId)
-#   ROOT_GROUP_ID         Keycloak group id of the RBAC root container
-#   PAYLOADS_FILE         host-side file, one `<app_id> <mapper_name> <base64 mapper JSON>` line per app
-#   GROUP_ID_PLACEHOLDER  token in the mapper JSON that stands for the app group id
-#   KC_WORKERS            apps reconciled in parallel
-#   KC_SYNC_ATTEMPTS      mapper sync attempts per app
-#   KC_SYNC_DELAY         seconds between two sync attempts
-#   KC_SYNC_TIMEOUT       seconds one sync attempt may take
-#
-# Output: one `<step>-failed <app_id>` line per app that failed.
-# Exit: 0 every app reconciled, 2 only mapper syncs failed, 1 otherwise.
-set -o pipefail
-: "${KC_CONTAINER:?KC_CONTAINER is required}"
-: "${KC_REALM:?KC_REALM is required}"
-: "${LDAP_COMPONENT_ID:?LDAP_COMPONENT_ID is required}"
-: "${ROOT_GROUP_ID:?ROOT_GROUP_ID is required}"
-: "${PAYLOADS_FILE:?PAYLOADS_FILE is required}"
-: "${GROUP_ID_PLACEHOLDER:?GROUP_ID_PLACEHOLDER is required}"
-: "${KC_WORKERS:?KC_WORKERS is required}"
-: "${KC_SYNC_ATTEMPTS:?KC_SYNC_ATTEMPTS is required}"
-: "${KC_SYNC_DELAY:?KC_SYNC_DELAY is required}"
-: "${KC_SYNC_TIMEOUT:?KC_SYNC_TIMEOUT is required}"
-[ -r "$PAYLOADS_FILE" ] || { echo "PAYLOADS_FILE not readable: $PAYLOADS_FILE" >&2; exit 1; }
-awk 'NF != 3 { bad = 1 } END { exit bad }' "$PAYLOADS_FILE" \
-  || { echo "PAYLOADS_FILE has a line without exactly 3 fields: $PAYLOADS_FILE" >&2; exit 1; }
-
-KCADM=/opt/keycloak/bin/kcadm.sh
-work="$(mktemp -d)"
-trap 'rm -rf "$work"' EXIT
-export KCADM work
-
-if ! container exec -i "$KC_CONTAINER" "$KCADM" get "groups/$ROOT_GROUP_ID/children" \
-  -r "$KC_REALM" -q max=500 --fields id,name --format csv --noquotes \
-  </dev/null >"$work/groups.csv"; then
-  echo "listing-failed groups/$ROOT_GROUP_ID/children"
-  exit 1
-fi
-
-if ! container exec -i "$KC_CONTAINER" "$KCADM" get components -r "$KC_REALM" \
-  -q "parentId=$LDAP_COMPONENT_ID" -q providerId=group-ldap-mapper \
-  -q max=500 --fields id,name --format csv --noquotes \
-  </dev/null >"$work/mappers.csv"; then
-  echo "listing-failed components"
-  exit 1
-fi
-
-# shellcheck disable=SC2329,SC2317  # invoked indirectly through reconcile_steps
-kcw() {
-  container exec -i "$KC_CONTAINER" "$KCADM" "$@" --config "$cfg"
-}
-
-# Args: app_id, mapper_name, base64 mapper JSON. Sets failed_step on failure.
-# shellcheck disable=SC2329,SC2317  # invoked indirectly through reconcile_app
-reconcile_steps() {
-  local app="$1" mapper_name="$2" payload_b64="$3"
-  local payload="$work/$1.json" group_id mapper_id stale_id attempt=1
-
-  group_id="$(awk -F',' -v n="$app" '!/^\[/ && !/Cgroup/ && $2==n {print $1; exit}' \
-    "$work/groups.csv" | tr -d '\r')"
-  if [ -z "$group_id" ]; then
-    group_id="$(kcw create "groups/$ROOT_GROUP_ID/children" -r "$KC_REALM" -s "name=$app" -i \
-      </dev/null | grep -vE '(^\[|Cgroup)' | grep -E '^[0-9a-fA-F-]{8,}$' | head -1 | tr -d '\r')"
-  fi
-  [ -n "$group_id" ] || { failed_step=ensure; return 1; }
-
-  kcw get "groups/$group_id/children" -r "$KC_REALM" \
-    -q max=500 --fields id,name --format csv --noquotes </dev/null \
-    | awk -F',' '!/^\[/ && !/Cgroup/ && $1 != "" { print $1 }' \
-    | tr -d '\r' >"$work/$app.children"
-  while IFS= read -r stale_id; do
-    [ -z "$stale_id" ] && continue
-    kcw delete "groups/$stale_id" -r "$KC_REALM" </dev/null || true  # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
-  done <"$work/$app.children"
-
-  printf '%s' "$payload_b64" | base64 -d \
-    | sed "s|$GROUP_ID_PLACEHOLDER|$group_id|g" >"$payload" \
-    || { failed_step=render; return 1; }
-
-  mapper_id="$(awk -F',' -v n="$mapper_name" '!/^\[/ && !/Cgroup/ && $2==n {print $1; exit}' \
-    "$work/mappers.csv" | tr -d '\r')"
-  if [ -n "$mapper_id" ]; then
-    sed "s|\"name\":|\"id\": \"$mapper_id\",\n    \"name\":|" "$payload" \
-      | kcw update "components/$mapper_id" -r "$KC_REALM" -f - \
-      >/dev/null || true  # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
-  else
-    mapper_id="$(kcw create components -r "$KC_REALM" -f - -i <"$payload" \
-      | grep -vE '(^\[|Cgroup)' | grep -E '^[0-9a-fA-F-]{8,}$' | head -1 | tr -d '\r')"
-  fi
-  [ -n "$mapper_id" ] || { failed_step=upsert; return 1; }
-
-  until timeout "$KC_SYNC_TIMEOUT" container exec -i "$KC_CONTAINER" "$KCADM" create \
-    "user-storage/$LDAP_COMPONENT_ID/mappers/$mapper_id/sync?direction=fedToKeycloak" \
-    -r "$KC_REALM" --config "$cfg" </dev/null >"$work/$app.sync" 2>&1; do
-    if [ "$attempt" -ge "$KC_SYNC_ATTEMPTS" ]; then
-      sed "s|^|$app sync: |" "$work/$app.sync" >&2
-      failed_step=sync
-      return 1
-    fi
-    attempt=$((attempt + 1))
-    sleep "$KC_SYNC_DELAY"
-  done
-}
-
-# Args: app_id, mapper_name, base64 mapper JSON. Leaves `<app_id>.done` or `<app_id>.failed` in $work.
-# shellcheck disable=SC2329,SC2317  # invoked indirectly through xargs + bash -c
-reconcile_app() {
-  local app="$1" cfg="/tmp/kcadm-$1-$$.config" failed_step=""
-  # shellcheck disable=SC2016  # $HOME and $1 expand in the container shell
-  if container exec -i "$KC_CONTAINER" sh -c 'cp "$HOME/.keycloak/kcadm.config" "$1"' sh "$cfg" </dev/null; then
-    reconcile_steps "$@"
-    container exec -i "$KC_CONTAINER" rm -f "$cfg" </dev/null
-  else
-    failed_step=config
-  fi
-  if [ -n "$failed_step" ]; then
-    printf '%s\n' "$failed_step" >"$work/$app.failed"
-  else
-    : >"$work/$app.done"
-  fi
-}
-export -f kcw reconcile_steps reconcile_app
-
-# shellcheck disable=SC2016  # the single-quoted body expands inside the child bash
-xargs -r -n 3 -P "$KC_WORKERS" bash -c 'reconcile_app "$@"' _ <"$PAYLOADS_FILE"
-xargs_rc=$?
-
-hard=0
-sync=0
-while read -r app _ || [ -n "$app" ]; do
-  [ -e "$work/$app.done" ] && continue
-  step=incomplete
-  [ -s "$work/$app.failed" ] && step="$(<"$work/$app.failed")"
-  printf '%s-failed %s\n' "$step" "$app"
-  if [ "$step" = sync ]; then sync=1; else hard=1; fi
-done <"$PAYLOADS_FILE"
-
-if [ "$xargs_rc" -ne 0 ]; then
-  echo "xargs exited $xargs_rc" >&2
-  hard=1
-fi
-[ "$hard" -eq 1 ] && exit 1
-[ "$sync" -eq 1 ] && exit 2
-exit 0

roles/web-app-keycloak/files/shell/per_app_mapper/resolve_root_group_id.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-# Resolve the Keycloak group id for the RBAC root path (e.g. /roles).
-# Echoes the id on stdout; empty when the path has no group; exits
-# non-zero when the groups listing itself cannot be fetched.
-#
-# Required env:
-#   KC_CONTAINER         keycloak container name
-#   KC_REALM             realm name
-#   KC_RBAC_ROOT_PATH    e.g. /roles
-set -eo pipefail
-: "${KC_CONTAINER:?KC_CONTAINER is required}"
-: "${KC_REALM:?KC_REALM is required}"
-: "${KC_RBAC_ROOT_PATH:?KC_RBAC_ROOT_PATH is required}"
-
-container exec -i "$KC_CONTAINER" /opt/keycloak/bin/kcadm.sh get groups \
-  -r "$KC_REALM" \
-  -q max=500 --fields id,path --format csv --noquotes \
-  > /tmp/kc_groups_top.txt
-
-awk -F',' -v p="$KC_RBAC_ROOT_PATH" \
-  '!/^\[/ && !/Cgroup/ && $2==p {print $1; exit}' \
-  /tmp/kc_groups_top.txt | tr -d '\r'

roles/web-app-keycloak/filter_plugins/rbac_per_app.py

@@ -8,7 +8,7 @@ per-app `group-ldap-mapper` setup:
 
 These names are referenced from `vars/main.yml`
 (`KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_PAYLOADS`), which hands them to the
-host-side `reconcile_apps.sh` script. Centralising them here means a future
+`files/python/rbac_reconcile.py` REST client. Centralising them here means a future
 rename of the mapper convention is a one-line change in this plugin
 and stays consistent everywhere.
 """

roles/web-app-keycloak/tasks/update/05_per_app_mappers.yml

@@ -23,10 +23,7 @@
 #
 # The shared default `ldap-roles` mapper (created by update/05_ldap/main.yml)
 # is removed here so it does not shadow the per-app mappers, and any
-# Keycloak groups it had imported flat under /roles are pruned. Each
-# step delegates to a dedicated `.sh` under
-# `files/shell/per_app_mapper/` so the shell logic lives outside the
-# YAML structure and stays easy to lint and reuse.
+# Keycloak groups it had imported flat under /roles are pruned.
 
 - name: "🚿 per-app mappers: flush pending handlers before mutating the shared mapper"
   ansible.builtin.meta: flush_handlers
@@ -37,81 +34,38 @@
       all_deployed: "{{ KEYCLOAK_RBAC_ALL_APP_IDS }}"
       current_run: "{{ KEYCLOAK_RBAC_RUN_APP_IDS }}"
 
-- name: "🗑️ per-app mappers: delete shared 'ldap-roles' mapper if present"
-  ansible.builtin.script:
-    cmd: "files/shell/per_app_mapper/delete_default_ldap_roles_mapper.sh"
-    executable: /bin/bash
-  environment:
-    KC_CONTAINER: "{{ KEYCLOAK_TARGET_CONTAINER }}"
-    KC_REALM: "{{ KEYCLOAK_REALM }}"
-    LDAP_COMPONENT_ID: "{{ ldap_cmp_id | trim }}"
-  changed_when: true
-  failed_when: false
-  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
-
-- name: "🧹 Purge stale flat / orphan groups under /<KEYCLOAK_RBAC_GROUP_NAME>"
-  ansible.builtin.script:
-    cmd: "files/shell/per_app_mapper/purge/root_orphans.sh"
-    executable: /bin/bash
-  environment:
-    KC_CONTAINER: "{{ KEYCLOAK_TARGET_CONTAINER }}"
-    KC_REALM: "{{ KEYCLOAK_REALM }}"
-    KC_RBAC_ROOT_PATH: "/{{ KEYCLOAK_RBAC_GROUP_NAME | trim('/') }}"
-    DEPLOYED_APP_IDS: "{{ KEYCLOAK_RBAC_ALL_APP_IDS | join(' ') }}"
-    KC_PURGE_WORKERS: "{{ KEYCLOAK_KCADM_WORKERS }}"
-  changed_when: true
-  failed_when: false
-  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
-
-- name: "🔍 per-app mappers: resolve the root group id once, it is loop-invariant"
-  ansible.builtin.script:
-    cmd: "files/shell/per_app_mapper/resolve_root_group_id.sh"
-    executable: /bin/bash
-  environment:
-    KC_CONTAINER: "{{ KEYCLOAK_TARGET_CONTAINER }}"
-    KC_REALM: "{{ KEYCLOAK_REALM }}"
-    KC_RBAC_ROOT_PATH: "/{{ KEYCLOAK_RBAC_GROUP_NAME | trim('/') }}"
-  register: per_app_root_id
-  changed_when: false
-  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
-
-- name: "✅ per-app mappers: assert root group was resolved"
-  ansible.builtin.assert:
-    that: (per_app_root_id.stdout | default('') | trim) | length > 0
-    fail_msg: "could not resolve /{{ KEYCLOAK_RBAC_GROUP_NAME | trim('/') }} group id"
-
-- name: "🔧 per-app mappers: set root group id fact"
-  ansible.builtin.set_fact:
-    kc_root_group_id: "{{ (per_app_root_id.stdout | default('')) | trim }}"
-  changed_when: false
-
 - name: "📄 per-app mappers: render the mapper payloads of every rbac app in current run"
   ansible.builtin.copy:
     dest: "{{ KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_PAYLOADS_FILE }}"
     mode: "0600"
     content: "{{ KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_PAYLOADS }}"
   changed_when: false
+  delegate_to: "{{ KEYCLOAK_HOST_NODE }}"
   no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
 
-- name: "✅ per-app mappers: ensure Keycloak group and mapper per rbac app in current run"
-  ansible.builtin.script:
-    cmd: "files/shell/per_app_mapper/reconcile_apps.sh"
-    executable: /bin/bash
-  environment:
-    KC_CONTAINER: "{{ KEYCLOAK_TARGET_CONTAINER }}"
-    KC_REALM: "{{ KEYCLOAK_REALM }}"
-    LDAP_COMPONENT_ID: "{{ ldap_cmp_id | trim }}"
-    ROOT_GROUP_ID: "{{ kc_root_group_id }}"
-    PAYLOADS_FILE: "{{ KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_PAYLOADS_FILE }}"
-    GROUP_ID_PLACEHOLDER: "{{ KEYCLOAK_PER_APP_GROUP_ID_PLACEHOLDER }}"
-    KC_WORKERS: "{{ KEYCLOAK_KCADM_WORKERS }}"
-    KC_SYNC_ATTEMPTS: "3"
-    KC_SYNC_DELAY: "20"
-    KC_SYNC_TIMEOUT: "{{ lookup('timeout', 60) | int }}"
-  register: per_app_reconcile
-  changed_when: true
-  failed_when: per_app_reconcile.rc not in [0, 1, 2]
-  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
+- name: "✅ per-app mappers: reconcile groups and mappers of every rbac app over REST"
+  ansible.builtin.include_tasks:
+    file: "{{ role_path }}/tasks/utils/rest_script.yml"
+    apply:
+      delegate_to: "{{ KEYCLOAK_HOST_NODE }}"
+  vars:
+    rest_script: rbac_reconcile.py
+    rest_script_env:
+      LDAP_COMPONENT_ID: "{{ ldap_cmp_id | trim }}"
+      RBAC_ROOT_PATH: "/{{ KEYCLOAK_RBAC_GROUP_NAME | trim('/') }}"
+      DEPLOYED_APP_IDS: "{{ KEYCLOAK_RBAC_ALL_APP_IDS | join(' ') }}"
+      PAYLOADS_FILE: "{{ KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_PAYLOADS_FILE }}"
+      WORKERS: "{{ KEYCLOAK_RBAC_WORKERS }}"
+      SYNC_ATTEMPTS: "3"
+      SYNC_DELAY: "20"
+      SYNC_TIMEOUT: "{{ lookup('timeout', 60) | int }}"
+    rest_script_ok_rcs: [0, 1, 2]
+    rest_script_changed: "synced=[1-9]"
+    rest_script_register: per_app_reconcile
+
+- name: "📊 per-app mappers: report how many rbac apps were synced"
+  ansible.builtin.debug:
+    msg: "{{ per_app_reconcile.stdout_lines | select('match', '^summary ') | list }}"
 
 - name: "⚠️ per-app mappers: report the rbac apps that failed"
   ansible.builtin.fail:

roles/web-app-keycloak/tasks/utils/rest_script.yml

@@ -0,0 +1,72 @@
+---
+# Param: rest_script          - script under files/python/ that imports keycloak_admin.py
+# Param: rest_script_env      - script environment besides KC_URL, KC_PID, KC_REALM and PYTHONPATH
+# Param: rest_script_ok_rcs   - exit codes that do not fail the run
+# Param: rest_script_changed  - regex on stdout that marks the run changed
+# Param: rest_script_register - fact the command result is stored under
+- name: "🔍 Keycloak REST | Resolve the host PID of the Keycloak container"
+  ansible.builtin.command:
+    argv:
+      - container
+      - inspect
+      - --type
+      - container
+      - --format
+      - "{{ '{{' }}.State.Pid{{ '}}' }}"
+      - "{{ KEYCLOAK_LOCAL_CID }}" # nocheck: container-exec-resolver  KEYCLOAK_LOCAL_CID resolved at delegate_to runtime; container_address would return a manager-only subshell
+  register: kc_rest_script_pid
+  changed_when: false
+
+- name: "📁 Keycloak REST | Create the staging directory of {{ rest_script }}"
+  ansible.builtin.tempfile:
+    state: directory
+    suffix: -keycloak-rest
+  register: kc_rest_script_dir
+  changed_when: false
+
+- name: "🐍 Keycloak REST | Stage, run and remove {{ rest_script }}"
+  block:
+    - name: "📄 Keycloak REST | Stage {{ rest_script }} with its client module"
+      ansible.builtin.copy:
+        src: "python/{{ kc_rest_script_file }}"
+        dest: "{{ kc_rest_script_dir.path }}/{{ kc_rest_script_file }}"
+        mode: "0600"
+      loop:
+        - keycloak_admin.py
+        - "{{ rest_script }}"
+      loop_control:
+        loop_var: kc_rest_script_file
+      changed_when: false
+
+    - name: "▶️ Keycloak REST | Run {{ rest_script }} in the Keycloak network namespace"
+      ansible.builtin.command:
+        argv:
+          - nsenter
+          - --target
+          - "{{ kc_rest_script_pid.stdout | trim }}"
+          - --net
+          - --
+          - python3
+          - "{{ kc_rest_script_dir.path }}/{{ rest_script }}"
+      environment: "{{ rest_script_env | combine(kc_rest_script_base_env) }}"
+      vars:
+        kc_rest_script_base_env:
+          KC_URL: "{{ KEYCLOAK_SERVER_INTERNAL_URL }}"
+          KC_PID: "{{ kc_rest_script_pid.stdout | trim }}"
+          KC_REALM: "{{ KEYCLOAK_REALM }}"
+          PYTHONPATH: "{{ kc_rest_script_dir.path }}"
+      register: kc_rest_script_result
+      changed_when: kc_rest_script_result.stdout is search(rest_script_changed)
+      failed_when: kc_rest_script_result.rc not in rest_script_ok_rcs
+      no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
+
+    - name: "📌 Keycloak REST | Store the result of {{ rest_script }}"
+      ansible.builtin.set_fact:
+        "{{ rest_script_register }}": "{{ kc_rest_script_result }}"
+      no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
+  always:
+    - name: "🗑️ Keycloak REST | Remove the staging directory of {{ rest_script }}"
+      ansible.builtin.file:
+        path: "{{ kc_rest_script_dir.path }}"
+        state: absent
+      changed_when: false

roles/web-app-keycloak/vars/main.yml

@@ -17,33 +17,40 @@ KEYCLOAK_RBAC_GROUP_NAME: "{{ RBAC.GROUP.NAME }}"
 KEYCLOAK_RBAC_ALL_APP_IDS: "{{ lookup('deployment').groups | rbac_app_ids }}"
 KEYCLOAK_RBAC_RUN_APP_IDS: "{{ lookup('deployment').running | rbac_app_ids }}"
 
-KEYCLOAK_KCADM_WORKERS: "{{ [RESOURCE_HOST_CPUS | int // 2, 1] | max }}"
-KEYCLOAK_PER_APP_GROUP_ID_PLACEHOLDER: "__KC_APP_GROUP_ID__"
+KEYCLOAK_RBAC_WORKERS: "{{ [RESOURCE_HOST_CPUS | int // 2, 1] | max }}"
+KEYCLOAK_RBAC_GROUP_LOGIC_DIGEST: "{{ lookup('file', lookup('path_absolute', 'roles/svc-db-openldap/filter_plugins/build_ldap_role_entries.py')) | hash('sha256') }}"
 KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_PAYLOADS_FILE: "/tmp/kc_per_app_mapper_payloads.txt"
-KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_BASE: >-
+KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_TEMPLATE: >-
   {{
-    ((KEYCLOAK_DICTIONARY_REALM.components['org.keycloak.storage.UserStorageProvider']
+    (KEYCLOAK_DICTIONARY_REALM.components['org.keycloak.storage.UserStorageProvider']
       | selectattr('name','equalto', KEYCLOAK_LDAP_CMP_NAME)
       | list | first
-     ).subComponents['org.keycloak.storage.ldap.mappers.LDAPStorageMapper']
-     | selectattr('name','equalto','ldap-roles')
-     | list | first
+    ).subComponents['org.keycloak.storage.ldap.mappers.LDAPStorageMapper']
+    | selectattr('name','equalto','ldap-roles')
+    | list | first
+  }}
+KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_BASE: >-
+  {{
+    (KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_TEMPLATE
      | dict2items
-     | rejectattr('key','in',['subComponents','id'])
+     | rejectattr('key','in',['subComponents','id','config'])
      | list | items2dict)
     | combine({
         'parentId':     ldap_cmp_id | trim,
         'providerId':   'group-ldap-mapper',
         'providerType': 'org.keycloak.storage.ldap.mappers.LDAPStorageMapper',
-        'config': {
-          'preserve.group.inheritance':            ['false'],
-          'group.name.ldap.attribute':             ['description'],
-          'groups.path.group':                     [KEYCLOAK_PER_APP_GROUP_ID_PLACEHOLDER],
-          'drop.non.existing.groups.during.sync':  ['true'],
-        }
-      }, recursive=True)
+        'config': (KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_TEMPLATE.config
+                   | dict2items
+                   | rejectattr('key','equalto','groups.path.group')
+                   | list | items2dict)
+                  | combine({
+                      'preserve.group.inheritance':            ['false'],
+                      'group.name.ldap.attribute':             ['description'],
+                      'drop.non.existing.groups.during.sync':  ['true'],
+                    }),
+      })
   }}
-# One `<app_id> <mapper_name> <base64 mapper JSON>` line per rbac app of the current run.
+# One `<app_id> <mapper_name> <base64 mapper JSON> <fingerprint>` line per rbac app of the current run.
 KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_PAYLOADS: |-
   {% for app_id in KEYCLOAK_RBAC_RUN_APP_IDS -%}
   {%- set mapper = KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_BASE | combine({
@@ -53,7 +60,14 @@ KEYCLOAK_PER_APP_LDAP_ROLES_MAPPER_PAYLOADS: |-
           'groups.path':        ['/' ~ (KEYCLOAK_RBAC_GROUP_NAME | trim('/')) ~ '/' ~ app_id],
         }
       }, recursive=True) -%}
-  {{ app_id }} {{ mapper['name'] }} {{ mapper | to_nice_json | b64encode }}
+  {%- set app = lookup('applications')[app_id] -%}
+  {%- set fingerprint = {
+        'mapper': mapper,
+        'rbac': app.get('rbac'),
+        'domains': app.get('domains'),
+        'logic': KEYCLOAK_RBAC_GROUP_LOGIC_DIGEST,
+      } | to_json(sort_keys=True) | hash('sha256') -%}
+  {{ app_id }} {{ mapper['name'] }} {{ mapper | to_nice_json | b64encode }} {{ fingerprint }}
   {% endfor %}
 
 KEYCLOAK_HEALTH_ENABLED: true

tests/unit/python/roles/web-app-keycloak/files/__init__.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+PROJECT_ROOT: Path = Path(__file__).resolve().parents[6]
+
+__all__ = ["PROJECT_ROOT"]

tests/unit/python/roles/web-app-keycloak/files/test_keycloak_admin.py

@@ -0,0 +1,79 @@
+import io
+import json
+import sys
+import tempfile
+import unittest
+import urllib.error
+from pathlib import Path
+from unittest import mock
+
+from . import PROJECT_ROOT
+
+sys.path.insert(0, str(PROJECT_ROOT / "roles/web-app-keycloak/files/python"))
+
+import keycloak_admin as m  # noqa: E402
+
+
+class TestContainerCredentials(unittest.TestCase):
+    def test_credentials_come_from_the_container_environment(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            environ = Path(tmp) / "environ"
+            environ.write_bytes(
+                b"PATH=/bin\0KEYCLOAK_PERMANENT_ADMIN_USERNAME=admin\0"
+                b"KEYCLOAK_PERMANENT_ADMIN_PASSWORD=a=b\0"
+            )
+            self.assertEqual(m.container_credentials(environ), ("admin", "a=b"))
+
+
+class _Response(io.BytesIO):
+    def __init__(self, body=b"", location=""):
+        super().__init__(body)
+        self.headers = {"Location": location} if location else {}
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+
+class TestKeycloakClient(unittest.TestCase):
+    def test_one_token_serves_calls_until_it_nears_expiry(self):
+        token = json.dumps({"access_token": "t", "expires_in": 300}).encode()
+        calls = []
+
+        def urlopen(request, timeout):
+            calls.append(request.full_url)
+            if request.full_url.endswith("/token"):
+                return _Response(token)
+            return _Response(b"", "http://kc/admin/realms/r/groups/new-id")
+
+        kc = m.Keycloak("http://kc", "r", "admin", "pw")
+        with mock.patch.object(m.urllib.request, "urlopen", urlopen):
+            _, created = kc.call("POST", "groups/root/children", {"name": "x"})
+            kc.call("GET", "groups/new-id")
+        self.assertEqual(created, "new-id")
+        self.assertEqual(sum(url.endswith("/token") for url in calls), 1)
+
+    def test_an_http_error_names_the_status_but_not_the_body(self):
+        token = json.dumps({"access_token": "t", "expires_in": 300}).encode()
+
+        def urlopen(request, timeout):
+            if request.full_url.endswith("/token"):
+                return _Response(token)
+            raise urllib.error.HTTPError(
+                request.full_url, 409, "Conflict", {}, io.BytesIO(b"secret detail")
+            )
+
+        kc = m.Keycloak("http://kc", "r", "admin", "pw")
+        with (
+            mock.patch.object(m.urllib.request, "urlopen", urlopen),
+            self.assertRaises(m.KeycloakError) as raised,
+        ):
+            kc.call("POST", "groups/root/children", {"name": "x"})
+        self.assertIn("HTTP 409", str(raised.exception))
+        self.assertNotIn("secret", str(raised.exception))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/python/roles/web-app-keycloak/files/test_rbac_reconcile.py

@@ -0,0 +1,226 @@
+import base64
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+from . import PROJECT_ROOT
+
+sys.path.insert(0, str(PROJECT_ROOT / "roles/web-app-keycloak/files/python"))
+
+import rbac_reconcile as m  # noqa: E402
+
+LDAP = "ldap-1"
+
+
+class FakeKeycloak:
+    def __init__(self):
+        self.groups = {"root": {"id": "root", "name": "roles", "attributes": {}}}
+        self.parents = {}
+        self.components = {}
+        self.syncs = []
+        self.failing_syncs = set()
+        self._next = 0
+
+    def _id(self, prefix):
+        self._next += 1
+        return f"{prefix}-{self._next}"
+
+    def add_group(self, name, parent="root", attributes=None):
+        group_id = self._id("g")
+        self.groups[group_id] = {
+            "id": group_id,
+            "name": name,
+            "attributes": attributes or {},
+        }
+        self.parents[group_id] = parent
+        return group_id
+
+    def add_mapper(self, name, config):
+        mapper_id = self._id("c")
+        self.components[mapper_id] = {
+            "id": mapper_id,
+            "name": name,
+            "parentId": LDAP,
+            "config": config,
+        }
+        return mapper_id
+
+    def children(self, group_id):
+        return [
+            dict(g)
+            for gid, g in self.groups.items()
+            if self.parents.get(gid) == group_id
+        ]
+
+    def call(self, method, path, payload=None, timeout=60):
+        route = path.split("?", 1)[0]
+        if method == "GET" and route == "components":
+            return [dict(c) for c in self.components.values()], ""
+        if method == "GET" and route == "group-by-path/roles":
+            return dict(self.groups["root"]), ""
+        if method == "POST" and route == "components":
+            mapper_id = self._id("c")
+            self.components[mapper_id] = {**payload, "id": mapper_id}
+            return None, mapper_id
+        if method == "PUT" and route.startswith("components/"):
+            self.components[route.split("/")[1]] = dict(payload)
+            return None, ""
+        if method == "DELETE" and route.startswith("components/"):
+            del self.components[route.split("/")[1]]
+            return None, ""
+        if (
+            method == "POST"
+            and route.startswith("groups/")
+            and route.endswith("/children")
+        ):
+            return None, self.add_group(payload["name"], route.split("/")[1])
+        if method == "GET" and route.startswith("groups/"):
+            return json.loads(json.dumps(self.groups[route.split("/")[1]])), ""
+        if method == "PUT" and route.startswith("groups/"):
+            self.groups[route.split("/")[1]] = dict(payload)
+            return None, ""
+        if method == "DELETE" and route.startswith("groups/"):
+            del self.groups[route.split("/")[1]]
+            return None, ""
+        if method == "POST" and route.startswith(f"user-storage/{LDAP}/mappers/"):
+            mapper_id = route.split("/")[3]
+            if mapper_id in self.failing_syncs:
+                raise m.KeycloakError("POST sync: HTTP 500")
+            self.syncs.append(mapper_id)
+            return None, ""
+        raise AssertionError(f"unexpected call {method} {path}")
+
+
+def _payload(app_id, filter_suffix=""):
+    return json.dumps(
+        {
+            "name": f"ldap-roles-{app_id}",
+            "parentId": LDAP,
+            "providerId": "group-ldap-mapper",
+            "config": {
+                "groups.path": [f"/roles/{app_id}"],
+                "groups.ldap.filter": [f"(cn={app_id}-*){filter_suffix}"],
+            },
+        }
+    )
+
+
+def _app(app_id, fingerprint="fp-1", filter_suffix=""):
+    return m.App(
+        app_id, f"ldap-roles-{app_id}", _payload(app_id, filter_suffix), fingerprint
+    )
+
+
+def _reconciler(kc, attempts=2):
+    groups = {g["name"]: g["id"] for g in kc.children("root")}
+    mappers = {c["name"]: c for c in kc.components.values()}
+    return m.Reconciler(kc, LDAP, "root", groups, mappers, attempts, 0, 5)
+
+
+class TestReconciler(unittest.TestCase):
+    def test_a_new_app_gets_its_group_mapper_sync_and_fingerprint(self):
+        kc = FakeKeycloak()
+        self.assertEqual(_reconciler(kc).run(_app("web-app-a")), "synced")
+        group = next(g for g in kc.groups.values() if g["name"] == "web-app-a")
+        mapper = next(
+            c for c in kc.components.values() if c["name"] == "ldap-roles-web-app-a"
+        )
+        self.assertEqual(mapper["config"]["groups.path"], ["/roles/web-app-a"])
+        self.assertEqual(kc.syncs, [mapper["id"]])
+        self.assertEqual(group["attributes"][m.FINGERPRINT], ["fp-1"])
+
+    def test_an_unchanged_app_is_not_synced_again(self):
+        kc = FakeKeycloak()
+        _reconciler(kc).run(_app("web-app-a"))
+        self.assertEqual(_reconciler(kc).run(_app("web-app-a")), "unchanged")
+        self.assertEqual(len(kc.syncs), 1)
+
+    def test_a_new_fingerprint_syncs_again(self):
+        kc = FakeKeycloak()
+        _reconciler(kc).run(_app("web-app-a"))
+        self.assertEqual(_reconciler(kc).run(_app("web-app-a", "fp-2")), "synced")
+        self.assertEqual(len(kc.syncs), 2)
+
+    def test_a_drifted_mapper_is_updated_and_synced(self):
+        kc = FakeKeycloak()
+        _reconciler(kc).run(_app("web-app-a"))
+        self.assertEqual(
+            _reconciler(kc).run(_app("web-app-a", filter_suffix="(drift)")), "synced"
+        )
+        mapper = next(
+            c for c in kc.components.values() if c["name"] == "ldap-roles-web-app-a"
+        )
+        self.assertEqual(
+            mapper["config"]["groups.ldap.filter"], ["(cn=web-app-a-*)(drift)"]
+        )
+
+    def test_extra_config_keycloak_adds_does_not_count_as_drift(self):
+        kc = FakeKeycloak()
+        _reconciler(kc).run(_app("web-app-a"))
+        mapper = next(
+            c for c in kc.components.values() if c["name"] == "ldap-roles-web-app-a"
+        )
+        mapper["config"]["mapped.group.attributes"] = [""]
+        self.assertEqual(_reconciler(kc).run(_app("web-app-a")), "unchanged")
+
+    def test_a_failed_sync_records_no_fingerprint(self):
+        kc = FakeKeycloak()
+        group_id = kc.add_group("web-app-a")
+        mapper_id = kc.add_mapper(
+            "ldap-roles-web-app-a",
+            {
+                "groups.path": ["/roles/web-app-a"],
+                "groups.ldap.filter": ["(cn=web-app-a-*)"],
+            },
+        )
+        kc.failing_syncs.add(mapper_id)
+        with self.assertRaises(m.StepError) as raised:
+            _reconciler(kc).run(_app("web-app-a"))
+        self.assertEqual(raised.exception.step, "sync")
+        self.assertNotIn(m.FINGERPRINT, kc.groups[group_id]["attributes"])
+
+    def test_existing_group_attributes_survive_the_fingerprint(self):
+        kc = FakeKeycloak()
+        kc.add_group("web-app-a", attributes={"LDAP_ID": ["x"]})
+        _reconciler(kc).run(_app("web-app-a"))
+        group = next(g for g in kc.groups.values() if g["name"] == "web-app-a")
+        self.assertEqual(group["attributes"]["LDAP_ID"], ["x"])
+
+
+class TestHousekeeping(unittest.TestCase):
+    def test_orphans_go_and_app_groups_stay(self):
+        kc = FakeKeycloak()
+        keep = kc.add_group("web-app-a")
+        kc.add_group("web-app-a-administrator")
+        kc.add_group("administrator")
+        kept, failures = m.purge_orphans(kc, "root", {"web-app-a"}, 2)
+        self.assertEqual(kept, {"web-app-a": keep})
+        self.assertEqual(failures, [])
+        self.assertEqual(sorted(g["name"] for g in kc.children("root")), ["web-app-a"])
+
+    def test_the_shared_mapper_is_dropped(self):
+        kc = FakeKeycloak()
+        kc.add_mapper(m.SHARED_MAPPER, {})
+        per_app = kc.add_mapper("ldap-roles-web-app-a", {})
+        mappers = m.ldap_mappers(kc, LDAP)
+        self.assertEqual(m.drop_shared_mapper(kc, mappers), [])
+        self.assertEqual(list(kc.components), [per_app])
+        self.assertNotIn(m.SHARED_MAPPER, mappers)
+
+    def test_payload_lines_are_parsed(self):
+        encoded = base64.b64encode(_payload("web-app-a").encode()).decode()
+        with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as handle:
+            handle.write(f"web-app-a ldap-roles-web-app-a {encoded} fp-9\n\n")
+        apps = m.read_payloads(handle.name)
+        Path(handle.name).unlink()
+        self.assertEqual(
+            [(a.app_id, a.mapper_name, a.fingerprint) for a in apps],
+            [("web-app-a", "ldap-roles-web-app-a", "fp-9")],
+        )
+        self.assertEqual(json.loads(apps[0].payload)["name"], "ldap-roles-web-app-a")
+
+
+if __name__ == "__main__":
+    unittest.main()