Fix CI Docker-in-Docker deployment, introduce vfs storage-driver, add inner dockerd bootstrap, enable portable json-file logging when running inside a container, and update workflow triggers for multi-branch testing.

Includes:
- Rewrite of test-deploy workflow to use isolated inner dockerd with privileged mode.
- Switch logging drivers to 'json-file' when IS_CONTAINER=true for compatibility with non-systemd CI runners.
- Adjust Dockerfile to install docker CLI and simplify package setup.
- Improve inventory creation and deploy steps for CI stability.
- Fully compatible with Ansible 2.20 variable handling.

Conversation reference:
https://chatgpt.com/share/6930e285-9604-800f-aad8-7a81c928548c

.github/workflows/test-deploy.yml

@@ -3,8 +3,10 @@ name: Build & Test Infinito.Nexus CLI in Docker Container
 on:
   push:
     branches:
-      - master
       - main
+      - master
+      - develop
+      - "*"
   pull_request:
 
 jobs:
@@ -26,26 +28,56 @@ jobs:
         web-app-postmarks,
         web-app-socialhome,
         web-svc-xmpp,
+
     steps:
-      - name: Checkout repository
+      - name: Main Checkout repository
         uses: actions/checkout@v4
 
+      - name: Show Docker version
+        run: docker version
+
       - name: Build Docker image
         run: |
           docker build --network=host --pull -t infinito:latest .
 
-      # 1) First deploy: normal + debug
+      # 1) First deploy: normal + debug (inner dockerd with vfs)
       - name: First deploy (normal + debug)
         run: |
-          docker run --network=host --rm \
+          docker run --network=host --rm --privileged \
-            -v /var/run/docker.sock:/var/run/docker.sock \
             -e EXCLUDED_ROLES="$EXCLUDED_ROLES" \
             infinito:latest \
             /bin/sh -lc '
               set -e
+
+              echo ">>> Starting inner dockerd..."
+              dockerd --debug --host=unix:///var/run/docker.sock --storage-driver=vfs \
+                >/var/log/dockerd.log 2>&1 &
+
+              echo ">>> Waiting for inner Docker daemon..."
+              for i in $(seq 1 60); do
+                if docker info >/dev/null 2>&1; then
+                  echo ">>> Inner Docker daemon is up."
+                  break
+                fi
+                sleep 1
+              done
+
+              if ! docker info >/dev/null 2>&1; then
+                echo "ERROR: Inner Docker daemon did not start in time." >&2
+                echo "----------- dockerd.log (inside infinito) -----------" >&2
+                if [ -f /var/log/dockerd.log ]; then
+                  sed -n "1,200p" /var/log/dockerd.log >&2
+                else
+                  echo "dockerd.log not found" >&2
+                fi
+                echo "-----------------------------------------------------" >&2
+                exit 1
+              fi
+
+              echo ">>> Inner Docker daemon is up, proceeding with deploy."
               cd /opt/infinito-src
 
-              # Create inventory (also creates inventories/github-ci/.password if missing)
+              echo ">>> Create CI inventory (normal + debug)..."
               infinito create inventory inventories/github-ci \
                 --host localhost \
                 --exclude "$EXCLUDED_ROLES" \
@@ -54,22 +86,47 @@ jobs:
               INVENTORY_PATH="inventories/github-ci/servers.yml"
               VAULT_FILE="inventories/github-ci/.password"
 
-              # First deploy with debug
+              echo ">>> First deploy (normal + debug)..."
               infinito deploy "$INVENTORY_PATH" -T server -p "$VAULT_FILE" --debug --skip-tests
             '
 
-      # 2) Second deploy: reset + debug
+      # 2) Second deploy: reset + debug (same inner dockerd pattern, also vfs)
       - name: Second deploy (--reset --debug)
         run: |
-          docker run --network=host --rm \
+          docker run --network=host --rm --privileged \
-            -v /var/run/docker.sock:/var/run/docker.sock \
             -e EXCLUDED_ROLES="$EXCLUDED_ROLES" \
             infinito:latest \
             /bin/sh -lc '
               set -e
+
+              echo ">>> Starting inner dockerd..."
+              dockerd --debug --host=unix:///var/run/docker.sock --storage-driver=vfs \
+                >/var/log/dockerd.log 2>&1 &
+
+              echo ">>> Waiting for inner Docker daemon..."
+              for i in $(seq 1 60); do
+                if docker info >/dev/null 2>&1; then
+                  echo ">>> Inner Docker daemon is up."
+                  break
+                fi
+                sleep 1
+              done
+
+              if ! docker info >/dev/null 2>&1; then
+                echo "ERROR: Inner Docker daemon did not start in time." >&2
+                echo "----------- dockerd.log (inside infinito) -----------" >&2
+                if [ -f /var/log/dockerd.log ]; then
+                  sed -n "1,200p" /var/log/dockerd.log >&2
+                else
+                  echo "dockerd.log not found" >&2
+                fi
+                echo "-----------------------------------------------------" >&2
+                exit 1
+              fi
+
               cd /opt/infinito-src
 
-              # Rebuild inventory; .password will be reused if present
+              echo ">>> Recreate CI inventory (reset run)..."
               infinito create inventory inventories/github-ci \
                 --host localhost \
                 --exclude "$EXCLUDED_ROLES" \
@@ -78,20 +135,47 @@ jobs:
               INVENTORY_PATH="inventories/github-ci/servers.yml"
               VAULT_FILE="inventories/github-ci/.password"
 
+              echo ">>> Second deploy (--reset --debug)..."
               infinito deploy "$INVENTORY_PATH" -T server -p "$VAULT_FILE" --skip-tests --reset --debug
             '
 
-      # 3) Third deploy: async (no debug)
+      # 3) Third deploy: async (no debug, same inner dockerd, also vfs)
       - name: Third deploy (async deploy – no debug)
         run: |
-          docker run --network=host --rm \
+          docker run --network=host --rm --privileged \
-            -v /var/run/docker.sock:/var/run/docker.sock \
             -e EXCLUDED_ROLES="$EXCLUDED_ROLES" \
             infinito:latest \
             /bin/sh -lc '
               set -e
+
+              echo ">>> Starting inner dockerd..."
+              dockerd --debug --host=unix:///var/run/docker.sock --storage-driver=vfs \
+                >/var/log/dockerd.log 2>&1 &
+
+              echo ">>> Waiting for inner Docker daemon..."
+              for i in $(seq 1 60); do
+                if docker info >/dev/null 2>&1; then
+                  echo ">>> Inner Docker daemon is up."
+                  break
+                fi
+                sleep 1
+              done
+
+              if ! docker info >/dev/null 2>&1; then
+                echo "ERROR: Inner Docker daemon did not start in time." >&2
+                echo "----------- dockerd.log (inside infinito) -----------" >&2
+                if [ -f /var/log/dockerd.log ]; then
+                  sed -n "1,200p" /var/log/dockerd.log >&2
+                else
+                  echo "dockerd.log not found" >&2
+                fi
+                echo "-----------------------------------------------------" >&2
+                exit 1
+              fi
+
               cd /opt/infinito-src
 
+              echo ">>> Create/update inventory for async deploy..."
               infinito create inventory inventories/github-ci \
                 --host localhost \
                 --exclude "$EXCLUDED_ROLES" \
@@ -100,6 +184,6 @@ jobs:
               INVENTORY_PATH="inventories/github-ci/servers.yml"
               VAULT_FILE="inventories/github-ci/.password"
 
-              # Async-style deploy: no --debug, so some processes run in parallel
+              echo ">>> Third deploy (async, no debug)..."
-              infinito deploy "$INVENTORY_PATH" -T server -p "$VAULT_FILE" --skip-tests
+              infinito deploy "$INVENTORY_PATH" -T server -p "$VAULT_FILE" --skip-tests --async
             '

Dockerfile

@@ -1,6 +1,6 @@
 FROM archlinux:latest
 
-# 1) Update system and install build/runtime deps
+# 1) Pakete inkl. docker (damit docker CLI im Container vorhanden ist)
 RUN pacman -Syu --noconfirm \
       base-devel \
       git \
@@ -10,15 +10,16 @@ RUN pacman -Syu --noconfirm \
       alsa-lib \
       go \
       rsync \
+      docker \
     && pacman -Scc --noconfirm
 
-# 2) Stub out systemctl & yay so post-install hooks and AUR calls never fail
+# 2) systemctl & yay stubben
 RUN printf '#!/bin/sh\nexit 0\n' > /usr/bin/systemctl \
     && chmod +x /usr/bin/systemctl \
     && printf '#!/bin/sh\nexit 0\n' > /usr/bin/yay \
     && chmod +x /usr/bin/yay
 
-# 3) Build & install python-simpleaudio from AUR manually (as non-root)
+# 3) python-simpleaudio aus AUR
 RUN useradd -m aur_builder \
  && su aur_builder -c "git clone https://aur.archlinux.org/python-simpleaudio.git /home/aur_builder/psa && \
                     cd /home/aur_builder/psa && \
@@ -26,35 +27,32 @@ RUN useradd -m aur_builder \
  && pacman -U --noconfirm /home/aur_builder/psa/*.pkg.tar.zst \
  && rm -rf /home/aur_builder/psa
 
-# 4) Clone Kevin’s Package Manager and create its venv
+# 4) pkgmgr + venv
 ENV PKGMGR_REPO=/opt/package-manager \
     PKGMGR_VENV=/root/.venvs/pkgmgr
 
 RUN git clone https://github.com/kevinveenbirkenbach/package-manager.git $PKGMGR_REPO \
  && python -m venv $PKGMGR_VENV \
  && $PKGMGR_VENV/bin/pip install --upgrade pip \
- # install pkgmgr’s own deps + the ansible Python library so infinito import yaml & ansible.plugins.lookup work
  && $PKGMGR_VENV/bin/pip install --no-cache-dir -r $PKGMGR_REPO/requirements.txt ansible \
- # drop a thin wrapper so `pkgmgr` always runs inside that venv
  && printf '#!/bin/sh\n. %s/bin/activate\nexec python %s/main.py "$@"\n' \
            "$PKGMGR_VENV" "$PKGMGR_REPO" > /usr/local/bin/pkgmgr \
  && chmod +x /usr/local/bin/pkgmgr
 
-# 5) Ensure pkgmgr venv bin and user-local bin are on PATH
 ENV PATH="$PKGMGR_VENV/bin:/root/.local/bin:${PATH}"
 
-# 6) Copy local Infinito.Nexus source into the image for override
+# 6) Infinito.Nexus Quelle rein
 COPY . /opt/infinito-src
 
-# 7) Install Infinito.Nexus via pkgmgr (clone-mode https)
+# 7) Infinito via pkgmgr (shallow)
 RUN pkgmgr install infinito --clone-mode shallow
 
-# 8) Override installed Infinito.Nexus with local source and clean ignored files
+# 8) Override mit lokaler Quelle
 RUN INFINITO_PATH=$(pkgmgr path infinito) && \
     rm -rf "$INFINITO_PATH"/* && \
     rsync -a --delete --exclude='.git' /opt/infinito-src/ "$INFINITO_PATH"/
 
-# 9) Symlink the infinito script into /usr/local/bin so ENTRYPOINT works
+# 9) Symlink
 RUN INFINITO_PATH=$(pkgmgr path infinito) && \
     ln -sf "$INFINITO_PATH"/main.py /usr/local/bin/infinito && \
     chmod +x /usr/local/bin/infinito

roles/docker-container/templates/base.yml.j2

@@ -6,7 +6,7 @@
       - "{{ docker_compose.files.env }}"
 {% endif %}
     logging:
-      driver: journald
+      driver: {{ "json-file" if IS_CONTAINER | bool else 'journald' }}
 {% filter indent(4) %}
     {% include 'roles/docker-container/templates/resource.yml.j2' %}
 {% endfilter %}

roles/svc-db-redis/templates/service.yml.j2

@@ -6,7 +6,7 @@
     container_name: {{ application_id | get_entity_name }}-redis
     restart: {{ DOCKER_RESTART_POLICY }}
     logging:
-      driver: journald
+      driver: {{ "json-file" if IS_CONTAINER else 'journald' }}
     volumes:
       - redis:/data
     # Just save in memory and prevent huge redis_volumes

roles/sys-svc-rdbms/templates/services/mariadb.yml.j2

@@ -4,7 +4,7 @@
   {{ database_host }}:
     container_name: {{ application_id | get_entity_name }}-database
     logging:
-      driver: journald
+      driver: {{ "json-file" if IS_CONTAINER | bool else 'journald' }}
     image: {{ database_image }}:{{ database_version }}
     restart: {{ DOCKER_RESTART_POLICY }}
     env_file:

roles/web-app-jenkins/templates/docker-compose.yml.j2

@@ -6,5 +6,5 @@
         - "127.0.0.1:{{ ports.localhost.http[application_id] }}:8080"
       volumes:
         - jenkins_data:/var/jenkins_home
-      log_driver: journald
+      log_driver: {{ "json-file" if IS_CONTAINER | bool else 'journald' }}
 {% include 'roles/docker-compose/templates/networks.yml.j2' %}

roles/web-app-matrix/templates/docker-compose.yml.j2

@@ -5,7 +5,7 @@
     container_name: {{ MATRIX_SYNAPSE_NAME }}
     restart: {{ DOCKER_RESTART_POLICY }}
     logging:
-      driver: journald
+      driver: {{ "json-file" if IS_CONTAINER | bool else 'journald' }}
     volumes:
       - synapse_data:/data
       - {{ MATRIX_SYNAPSE_CONFIG_PATH_HOST }}:{{ MATRIX_SYNAPSE_CONFIG_PATH_CONTAINER }}:ro

roles/web-app-nextcloud/templates/docker-compose.yml.j2

@@ -5,7 +5,7 @@
     image: "{{ NEXTCLOUD_PROXY_IMAGE }}:{{ NEXTCLOUD_PROXY_VERSION }}"
     container_name: "{{ NEXTCLOUD_PROXY_CONTAINER }}"
     logging:
-      driver: journald
+      driver: {{ "json-file" if IS_CONTAINER | bool else 'journald' }}
     restart: {{ DOCKER_RESTART_POLICY }}
     ports:
       - "127.0.0.1:{{ NEXTCLOUD_PORT }}:{{ container_port }}"
@@ -99,7 +99,7 @@
     image: "{{ NEXTCLOUD_IMAGE }}:{{ NEXTCLOUD_VERSION }}"
     restart: {{ DOCKER_RESTART_POLICY }}
     logging:
-      driver: journald
+      driver: {{ "json-file" if IS_CONTAINER | bool else 'journald' }}
     volumes:
       - data:{{ NEXTCLOUD_DOCKER_WORK_DIRECTORY }}
     entrypoint: /cron.sh

roles/web-app-openproject/templates/docker-compose.yml.j2

@@ -1,7 +1,7 @@
 # @todo Test which containers can be removed crom cental_database networks
 x-op-app: &app
   logging:
-    driver: journald
+    driver: {{ "json-file" if IS_CONTAINER | bool else 'journald' }}
   image: {{ OPENPROJECT_CUSTOM_IMAGE }}
   {{ lookup('template', 'roles/docker-container/templates/build.yml.j2') | indent(2) }}