Merge branch 'master' of github.com:kevinveenbirkenbach/infinito-nexus

docs/guides/administrator/README.md

@@ -15,7 +15,7 @@ Follow these guides to install and configure Infinito.Nexus:
 - **Networking & VPN** - Configure `WireGuard`, `OpenVPN`, and `Nginx Reverse Proxy`.
 
 ## Managing & Updating Infinito.Nexus 🔄
-- Regularly update services using `update-docker`, `update-pacman`, or `update-apt`.
+- Regularly update services using `update-pacman`, or `update-apt`.
 - Monitor system health with `sys-ctl-hlth-btrfs`, `sys-ctl-hlth-webserver`, and `sys-ctl-hlth-docker-container`.
 - Automate system maintenance with `sys-lock`, `sys-ctl-cln-bkps`, and `sys-ctl-rpr-docker-hard`.
 

group_vars/all/07_services.yml

@@ -14,7 +14,6 @@ SYS_SERVICE_BACKUP_RMT_2_LOC:           "{{ 'svc-bkp-rmt-2-loc'         | get_se
 SYS_SERVICE_BACKUP_DOCKER_2_LOC:        "{{ 'sys-ctl-bkp-docker-2-loc'  | get_service_name(SOFTWARE_NAME) }}"
 SYS_SERVICE_REPAIR_DOCKER_SOFT:         "{{ 'sys-ctl-rpr-docker-soft'   | get_service_name(SOFTWARE_NAME) }}"
 SYS_SERVICE_REPAIR_DOCKER_HARD:         "{{ 'sys-ctl-rpr-docker-hard'   | get_service_name(SOFTWARE_NAME) }}"
-SYS_SERVICE_UPDATE_DOCKER:              "{{ 'update-docker'             | get_service_name(SOFTWARE_NAME) }}"
 
 ## On Failure
 SYS_SERVICE_ON_FAILURE_COMPOSE:         "{{ ('sys-ctl-alm-compose@') | get_service_name(SOFTWARE_NAME, False) }}%n.service"
@@ -48,8 +47,7 @@ SYS_SERVICE_GROUP_MANIPULATION: >
       SYS_SERVICE_GROUP_CLEANUP +
       SYS_SERVICE_GROUP_REPAIR +
       SYS_SERVICE_GROUP_OPTIMIZATION +
-      SYS_SERVICE_GROUP_MAINTANANCE +
+      SYS_SERVICE_GROUP_MAINTANANCE
-      [ SYS_SERVICE_UPDATE_DOCKER ]
     ) | sort
   }}
 

roles/pkgmgr/tasks/01_core.yml

@@ -43,3 +43,7 @@
     chdir: "{{ PKGMGR_INSTALL_PATH }}"
     executable: /bin/bash
   become: true
+
+- name: "Update all repositories with pkgmgr"
+  command: "pkgmgr pull --all"
+  when: MODE_UPDATE | bool

roles/update-compose/tasks/01_core.yml

@@ -14,13 +14,6 @@
     name: update-apt
   when: ansible_distribution == "Debian"
 
-- name: "Update Docker Images"
-  include_role:
-    name: update-docker
-  when:
-    - docker_compose_directory_stat.stat.exists
-    - run_once_update_docker is not defined
-
 - name: "Check if yay is installed"
   command: which yay
   register: yay_installed
@@ -51,7 +44,3 @@
   register: pkgmgr_available
   failed_when: false
 
-- name: "Update all repositories using pkgmgr"
-  include_role:
-    name: update-pkgmgr
-  when: pkgmgr_available.rc == 0

roles/update-docker/README.md

@@ -1,27 +0,0 @@
-# Update Docker
-
-## Description
-
-This role updates Docker Compose instances by checking for changes in Docker image digests and applying updates if necessary. It utilizes a Python script to handle git pulls and Docker image pulls, and rebuilds containers when changes are detected.
-
-## Overview
-
-The role performs the following:
-- Deploys a Python script to check for Docker image updates.
-- Configures a systemd service to run the update script.
-- Restarts the Docker update service upon configuration changes.
-- Supports additional procedures for specific Docker applications (e.g., Discourse, Mastodon, Nextcloud).
-
-## Purpose
-
-The role is designed to ensure that Docker images remain current by automatically detecting changes and rebuilding containers as needed. This helps maintain a secure and efficient container environment.
-
-## Features
-
-- **Docker Image Monitoring:** Checks for changes in image digests.
-- **Automated Updates:** Pulls new images and rebuilds containers when necessary.
-- **Service Management:** Configures and restarts a systemd service to handle updates.
-- **Application-Specific Procedures:** Includes hooks for updating specific Docker applications.
-
-## Credits 📝
-It was created with the help of ChatGPT. The conversation is available [here](https://chat.openai.com/share/165418b8-25fa-433b-baca-caded941e22a)

roles/update-docker/meta/main.yml

@@ -1,27 +0,0 @@
-galaxy_info:
-  author: "Kevin Veen-Birkenbach"
-  description: "Updates Docker Compose instances by detecting changes in Docker image digests and rebuilding containers when necessary. This role automates Docker image pulls and container rebuilds."
-  license: "Infinito.Nexus NonCommercial License"
-  license_url: "https://s.infinito.nexus/license"
-  company: |
-    Kevin Veen-Birkenbach
-    Consulting & Coaching Solutions
-    https://www.veen.world
-  min_ansible_version: "2.9"
-  platforms:
-  - name: Archlinux
-    versions:
-    - rolling
-  - name: Ubuntu
-    versions:
-    - all
-  galaxy_tags:
-  - docker
-  - update
-  - compose
-  - images
-  - systemd
-  - maintenance
-  repository: "https://s.infinito.nexus/code"
-  issue_tracker_url: "https://s.infinito.nexus/issues"
-  documentation: "https://docs.infinito.nexus"

roles/update-docker/tasks/01_core.yml

@@ -1,20 +0,0 @@
-- name: Include dependency 'sys-lock'
-  include_role:
-    name: sys-lock
-  when: run_once_sys_lock is not defined
-
-- name: "start {{ 'sys-ctl-bkp-docker-2-loc-everything' | get_service_name(SOFTWARE_NAME) }}"
-  systemd:
-    name: "{{ 'sys-ctl-bkp-docker-2-loc-everything' | get_service_name(SOFTWARE_NAME) }}"
-    state: started
-  when:
-  - MODE_BACKUP | bool
-
-- include_role:
-    name: sys-service
-  vars:
-    system_service_restarted:           true
-    system_service_timer_enabled:       false
-    system_service_tpl_on_failure:      "{{ SYS_SERVICE_ON_FAILURE_COMPOSE }}"
-    system_service_tpl_exec_start:      "{{ system_service_script_exec }} {{ PATH_DOCKER_COMPOSE_INSTANCES }}"
-    system_service_tpl_exec_start_pre:  "/usr/bin/python {{ PATH_SYSTEM_LOCK_SCRIPT }} {{ SYS_SERVICE_GROUP_MANIPULATION | join(' ')  }} --ignore {{ SYS_SERVICE_GROUP_CLEANUP | join(' ') }} {{ 'update-docker' | get_service_name(SOFTWARE_NAME) }} --timeout '{{ SYS_TIMEOUT_DOCKER_UPDATE }}'"

roles/update-docker/tasks/main.yml

@@ -1,4 +0,0 @@
-- block:
-    - include_tasks: 01_core.yml
-    - include_tasks: utils/run_once.yml
-  when: run_once_update_docker is not defined

roles/update-docker/templates/script.py.j2

@@ -1,217 +0,0 @@
-import os
-import subprocess
-import sys
-import time
-
-def run_command(command):
-    """
-    Executes the specified shell command, streaming and collecting its output in real-time.
-    If the command exits with a non-zero status, a subprocess.CalledProcessError is raised,
-    including the exit code, the executed command, and the full output (as bytes) for debugging purposes.
-    """
-    process = None
-    try:
-        process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        output = []
-
-        for line in iter(process.stdout.readline, b''):
-            decoded_line = line.decode()
-            output.append(decoded_line)
-            sys.stdout.write(decoded_line)
-
-        return_code = process.wait()
-        if return_code:
-            full_output = ''.join(output)
-            raise subprocess.CalledProcessError(return_code, command, output=full_output.encode())
-    finally:
-        if process and process.stdout:
-            process.stdout.close()
-
-def git_pull():
-    """
-    Checks whether the Git repository in the specified directory is up to date and performs a git pull if necessary.
-    
-    Raises:
-        Exception: If retrieving the local or remote git revision fails because the command returns a non-zero exit code.
-    """
-    print("Checking if the git repository is up to date.")
-
-    # Run 'git rev-parse @' and check its exit code explicitly.
-    local_proc = subprocess.run("git rev-parse @", shell=True, capture_output=True)
-    if local_proc.returncode != 0:
-        error_msg = local_proc.stderr.decode().strip() or "Unknown error while retrieving local revision."
-        raise Exception(f"Failed to retrieve local git revision: {error_msg}")
-    local = local_proc.stdout.decode().strip()
-
-    # Run 'git rev-parse @{u}' and check its exit code explicitly.
-    remote_proc = subprocess.run("git rev-parse @{u}", shell=True, capture_output=True)
-    if remote_proc.returncode != 0:
-        error_msg = remote_proc.stderr.decode().strip() or "Unknown error while retrieving remote revision."
-        raise Exception(f"Failed to retrieve remote git revision: {error_msg}")
-    remote = remote_proc.stdout.decode().strip()
-
-    if local != remote:
-        print("Repository is not up to date. Performing git pull.")
-        run_command("git pull")
-        return True
-
-    print("Repository is already up to date.")
-    return False
-
-{% raw %}        
-def get_image_digests(directory):
-    """
-    Retrieves the image digests for all images in the specified Docker Compose project.
-    """
-    compose_project = os.path.basename(directory)
-    try:
-        images_output = subprocess.check_output(
-            f'docker images --format "{{{{.Repository}}}}:{{{{.Tag}}}}@{{{{.Digest}}}}" | grep {compose_project}', 
-            shell=True
-        ).decode().strip()
-        return dict(line.split('@') for line in images_output.splitlines() if line)
-    except subprocess.CalledProcessError as e:
-        if e.returncode == 1:  # grep no match found
-            return {}
-        else:
-            raise  # Other errors are still raised
-{% endraw %}
-
-def is_any_service_up():
-    """
-    Checks if any Docker services are currently running.
-    """
-    process = subprocess.Popen("docker-compose ps -q", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    output, _ = process.communicate()
-    service_ids = output.decode().strip().splitlines()
-    return bool(service_ids)
-
-def pull_docker_images():
-    """
-    Pulls the latest Docker images for the project.
-    """
-    print("Pulling docker images.")
-    try:
-        run_command("docker-compose pull")
-    except subprocess.CalledProcessError as e:
-        if "pull access denied" in e.output.decode() or "must be built from source" in e.output.decode():
-            print("Need to build the image from source.")
-            return True
-        else:
-            print("Failed to pull images with unexpected error.")
-            raise
-    return False
-
-def update_docker(directory):
-    """
-    Checks for updates to Docker images and rebuilds containers if necessary.
-    """
-    print(f"Checking for updates to Docker images in {directory}.")        
-    before_digests = get_image_digests(directory)
-    need_to_build = pull_docker_images()
-    after_digests = get_image_digests(directory)
-    if before_digests != after_digests:
-        print("Changes detected in image digests. Rebuilding containers.")
-        need_to_build = True
-        
-    if need_to_build:
-        # This propably just rebuilds the Dockerfile image if there is a change in the other docker compose containers
-        run_command("docker-compose build --pull")
-        start_docker(directory)
-    else:
-        print("Docker images are up to date. No rebuild necessary.")
-
-def update_discourse(directory):
-    """
-    Updates Discourse by running the rebuild command on the launcher script.
-    """
-    docker_repository_directory   = os.path.join(directory, "services", "{{ applications | get_app_conf('web-app-discourse','repository') }}")
-    print(f"Using path {docker_repository_directory } to pull discourse repository.")
-    os.chdir(docker_repository_directory )
-    if git_pull():
-        print("Start Discourse update procedure.")
-        update_procedure("docker stop {{ applications | get_app_conf('web-app-discourse','docker.services.discourse.name') }}")
-        update_procedure("docker rm {{ applications | get_app_conf('web-app-discourse','docker.services.discourse.name') }}")
-        try:
-            update_procedure("docker network connect {{ applications | get_app_conf('web-app-discourse','docker.network') }} {{ applications | get_app_conf('svc-db-postgres', 'docker.network') }}")
-        except subprocess.CalledProcessError as e:
-            error_message = e.output.decode()
-            if "already exists" in error_message or "is already connected" in error_message:
-                print("Network connection already exists. Skipping...")
-            else:
-                raise
-        update_procedure("./launcher rebuild {{ applications | get_app_conf('web-app-discourse','docker.services.discourse.name') }}")
-    else:
-        print("Discourse update skipped. No changes in git repository.")
-
-def upgrade_listmonk():
-    """
-    Runs the upgrade for Listmonk
-    """
-    print("Starting Listmonk upgrade.")
-    run_command('echo "y" | docker compose run -T application ./listmonk --upgrade')
-    print("Upgrade complete.")
-
-def update_procedure(command):
-    """
-    Attempts to execute a command up to a maximum number of retries.
-    """
-    max_attempts = 3
-    for attempt in range(max_attempts):
-        try:
-            run_command(command)
-            break  # If the command succeeds, exit the loop
-        except subprocess.CalledProcessError as e:
-            if attempt < max_attempts - 1:  # Check if it's not the last attempt
-                print(f"Attempt {attempt + 1} failed, retrying in 60 seconds...")
-                time.sleep(60)  # Wait for 60 seconds before retrying
-            else:
-                print("All attempts to update have failed.")
-                raise  # Re-raise the last exception after all attempts fail
-
-def start_docker(directory):
-    """
-    Starts or restarts Docker services in the specified directory.
-    """
-    if is_any_service_up():
-        print(f"Restarting containers in {directory}.")
-        run_command("docker-compose up -d --force-recreate")
-    else:
-        print(f"Skipped starting. No service is up in {directory}.")
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("Please provide the path to the parent directory as a parameter.")
-        sys.exit(1)
-
-    parent_directory = sys.argv[1]
-    for dir_entry in os.scandir(parent_directory):
-        if dir_entry.is_dir():
-            dir_path = dir_entry.path
-            print(f"Checking for updates in: {dir_path}")
-            os.chdir(dir_path)
-            
-            # Pull git repository if it exist
-            # @deprecated: This function should be removed in the future, as soon as all docker applications use the correct folder path
-            if os.path.isdir(os.path.join(dir_path, ".git")):
-                print("DEPRECATED: Docker .git repositories should be saved under /opt/docker/{instance}/services/{repository_name} ")
-                git_pull()
-            
-            if os.path.basename(dir_path) == "matrix":
-                # No autoupdate for matrix is possible atm, 
-                # due to the reason that the role has to be executed every time.
-                # The update has to be executed in the role
-                # @todo implement in future
-                pass
-            else:
-                # Pull and update docker images
-                update_docker(dir_path)
-                
-                # The following instances need additional update and upgrade procedures
-                if os.path.basename(dir_path) == "discourse":
-                    update_discourse(dir_path)
-                elif os.path.basename(dir_path) == "listmonk":
-                    upgrade_listmonk()
-                    
-                # @todo implement dedicated procedure for bluesky
-                # @todo implement dedicated procedure for taiga

roles/update-docker/vars/main.yml

@@ -1,2 +0,0 @@
-application_id: update-docker
-system_service_id: "{{ application_id }}"

roles/update-pip/README.md

@@ -1,23 +0,0 @@
-# Update Pip Packages
-
-## Description
-
-This Ansible role automatically updates all installed Python Pip packages to their latest versions.
-
-## Overview
-
-The role performs the following:
-- Executes a command to retrieve all installed Python Pip packages.
-- Updates each package individually to its latest available version.
-- Ensures a smooth and automated Python environment maintenance process.
-
-## Purpose
-
-Ensures Python packages remain up-to-date, improving security and functionality.
-
-## Features
-
-- **Automatic Updates:** Automates the process of upgrading Python packages.
-- **Platform Independent:** Works on Linux, macOS, and Windows environments.
-- **Ansible Integration:** Easy to include in larger playbooks or maintenance routines.
-

roles/update-pip/meta/main.yml

@@ -1,25 +0,0 @@
-galaxy_info:
-  author: "Kevin Veen-Birkenbach"
-  license: "Infinito.Nexus NonCommercial License"
-  license_url: "https://s.infinito.nexus/license"
-  company: |
-    Kevin Veen-Birkenbach
-    Consulting & Coaching Solutions
-    https://www.veen.world
-  description: "Automatically updates all Python Pip packages to their latest available versions."
-  min_ansible_version: "2.9"
-  platforms:
-  - name: Ubuntu
-    versions:
-    - all
-  - name: Archlinux
-    versions:
-    - rolling
-  - name: Debian
-    versions:
-    - all
-  galaxy_tags:
-  - python
-  - pip
-  - update
-  - maintenance

roles/update-pip/tasks/main.yml

@@ -1,9 +0,0 @@
-- block:
-  - name: Include dependency 'dev-python-pip'
-    include_role:
-      name: dev-python-pip
-    when: run_once_dev_python_pip is not defined
-  - include_tasks: utils/run_once.yml
-    vars:
-      flush_handlers: false
-  when: run_once_update_pip is not defined

roles/update-pip/vars/main.yml

@@ -1 +0,0 @@
-application_id: update-pip

roles/update-pkgmgr/README.md

@@ -1,27 +0,0 @@
-# Update pkgmgr
-
-## Description
-
-This role checks if the [package manager](https://github.com/kevinveenbirkenbach/package-manager) is available on the system. If so, it runs `pkgmgr update --all` to update all repositories managed by the `pkgmgr`.
-
-## Overview
-
-This role performs the following tasks:
-- Checks if the `pkgmgr` command is available.
-- If available, runs `pkgmgr update --all` to update all repositories.
-
-## Purpose
-
-The purpose of this role is to simplify system updates by using the `pkgmgr` package manager to handle all repository updates with a single command.
-
-## Features
-
-- **Conditional Execution**: Runs only if the `pkgmgr` command is found on the system.
-- **Automated Updates**: Automatically runs `pkgmgr update --all` to update all repositories.
-
-## License
-
-Infinito.Nexus NonCommercial License  
-[Learn More](https://s.infinito.nexus/license)
-
-

roles/update-pkgmgr/TODO.md

@@ -1,2 +0,0 @@
-# Todos
-- Activate update again. Atm not possible, because it pulls all repos

roles/update-pkgmgr/meta/main.yml

@@ -1,24 +0,0 @@
----
-galaxy_info:
-  author: "Kevin Veen-Birkenbach"
-  description: "Checks if the pkgmgr command is available and runs 'pkgmgr update --all' to update all repositories."
-  license: "Infinito.Nexus NonCommercial License"
-  license_url: "https://s.infinito.nexus/license"
-  company: |
-    Kevin Veen-Birkenbach
-    Consulting & Coaching Solutions
-    https://www.veen.world
-  min_ansible_version: "2.9"
-  platforms:
-    - name: Linux
-      versions:
-        - all
-  galaxy_tags:
-    - update
-    - pkgmgr
-    - pkgmgr
-    - system
-  repository: "https://s.infinito.nexus/code"
-  issue_tracker_url: "https://s.infinito.nexus/issues"
-  documentation: "https://docs.infinito.nexus"
-dependencies: []

roles/update-pkgmgr/tasks/main.yml

@@ -1,3 +0,0 @@
-# run_once_update_pkgmgr: deactivated
-#- name: "Update all repositories with pkgmgr"
-#  command: "pkgmgr update --all"

roles/update-pkgmgr/vars/main.yml

@@ -1 +0,0 @@
-application_id: update-pkgmgr

roles/web-app-listmonk/tasks/main.yml

@@ -30,6 +30,14 @@
     chdir: "{{ docker_compose.directories.instance }}"
   when: "'No relations found.' in db_tables.stdout"
 
+- name: "Listmonk | run DB/schema upgrade (non-interactive)"
+  ansible.builtin.shell: |
+    set -o pipefail
+    echo "y" | docker compose run -T application ./listmonk --upgrade
+  args:
+    chdir: "{{ docker_compose.directories.instance }}"
+  when: MODE_UPDATE | bool
+
 - name: Build OIDC settings JSON
   set_fact:
     oidc_settings_json: >-
@@ -73,3 +81,4 @@
   no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
   async: "{{ ASYNC_TIME if ASYNC_ENABLED | bool else omit }}"
   poll:  "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}"
+

roles/web-app-matrix/tasks/01_docker.yml

@@ -67,7 +67,6 @@
   notify: docker compose up
 
 # Pull image when update is wished. 
-# @todo This should be moved to update-docker
 - name: docker compose pull
   command:
     cmd: docker-compose -p "{{ MATRIX_PROJECT }}" pull