Changed from jq to format

.gitignore

@@ -0,0 +1 @@
+databases.csv

README.md

@@ -1,5 +1,5 @@
-# docker-volume-backup
-[![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](./LICENSE.txt) [![Travis CI](https://travis-ci.org/kevinveenbirkenbach/docker-volume-backup.svg?branch=master)](https://travis-ci.org/kevinveenbirkenbach/docker-volume-backup)
+# Backup Docker Volumes to Local
+[![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](./LICENSE.txt)
 
 ## goal
 This script backups all docker-volumes with the help of rsync.
@@ -9,49 +9,59 @@ It is part of the following scheme:
 ![backup scheme](https://www.veen.world/wp-content/uploads/2020/12/server-backup-768x567.jpg)
 Further information you will find [in this blog post](https://www.veen.world/2020/12/26/how-i-backup-dedicated-root-servers/).
 
-## Backup
+## Backup all volumes
 Execute:
 
 ```bash
-./docker-volume-backup.sh
+./backup-docker-to-local.sh
 ```
 
 ## Recover
+
+### database
+```bash
+  docker exec -i mysql_container mysql -uroot -psecret database < db.sql
+```
+
+### volume
 Execute:
 
 ```bash
-./docker-volume-recover.sh {{volume_name}} {{backup_path}}
+
+bash ./recover-docker-from-local.sh "{{volume_name}}" "$(sha256sum /etc/machine-id | head -c 64)" "{{version_to_recover}}"
+
 ```
 
+### Database
+
 ## Debug
 To checkout what's going on in the mount container type in the following command:
 
 ```bash
 docker run -it --entrypoint /bin/sh --rm --volumes-from {{container_name}} -v /Backups/:/Backups/ kevinveenbirkenbach/alpine-rsync
 ```
-## Manual Backup
-rsync -aPvv  '***{{source_path}}***/' ***{{destination_path}}***";
 
-## Test
-Delete the volume.
+## Setup
+Install pandas
 
-```bash
-docker rm -f container-name
-docker volume rm volume-name
-```
+## Author
 
-Recover the volume:
+Kevin Veen-Birkenbach  
+- 📧 Email: [kevin@veen.world](mailto:kevin@veen.world)
+- 🌍 Website: [https://www.veen.world/](https://www.veen.world/)
 
-```bash
-docker volume create volume-name
-docker run --rm -v volume-name:/recover/ -v ~/backup/:/backup/ "kevinveenbirkenbach/alpine-rsync" sh -c "rsync -avv /backup/ /recover/"
-```
+## License
 
-Restart the container.
-
-## Optimation
-This setup script is not optimized yet for performance. Please optimized this script for performance if you want to use it in a professional environment.
+This project is licensed under the GNU Affero General Public License v3.0. The full license text is available in the `LICENSE` file of this repository.
 
 ## More information
+- https://docs.docker.com/storage/volumes/
 - https://blog.ssdnodes.com/blog/docker-backup-volumes/
 - https://www.baculasystems.com/blog/docker-backup-containers/
+- https://gist.github.com/spalladino/6d981f7b33f6e0afe6bb
+- https://stackoverflow.com/questions/26331651/how-can-i-backup-a-docker-container-with-its-data-volumes
+- https://netfuture.ch/2013/08/simple-versioned-timemachine-like-backup-using-rsync/
+- https://zwischenzugs.com/2016/08/29/bash-to-python-converter/
+- https://en.wikipedia.org/wiki/Incremental_backup#Incremental
+- https://unix.stackexchange.com/questions/567837/linux-backup-utility-for-incremental-backups
+- https://chat.openai.com/share/6d10f143-3f7c-4feb-8ae9-5644c3433a65

backup-docker-to-local.py

@@ -0,0 +1,290 @@
+#!/bin/python
+# Backups volumes of running containers
+
+import subprocess
+import os
+import re
+import pathlib
+import pandas
+from datetime import datetime
+import argparse
+
+class BackupException(Exception):
+    """Generic exception for backup errors."""
+    pass
+
+def execute_shell_command(command):
+    """Execute a shell command and return its output."""
+    print(command)
+    process = subprocess.Popen([command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+    out, err = process.communicate()
+    if process.returncode != 0:
+        raise BackupException(f"Error in command: {command}\nOutput: {out}\nError: {err}\nExit code: {process.returncode}")
+    return [line.decode("utf-8") for line in out.splitlines()]
+
+def create_version_directory():
+    """Create necessary directories for backup."""
+    version_dir = os.path.join(VERSIONS_DIR, BACKUP_TIME)
+    pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
+    return version_dir
+
+def get_machine_id():
+    """Get the machine identifier."""
+    return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
+
+### GLOBAL CONFIGURATION ###
+
+IMAGES_NO_STOP_REQUIRED = [
+    'akaunting',
+    'baserow',
+    'discourse',
+    'element',
+    'gitea',
+    'listmonk',
+    'mastodon',
+    'matomo',
+    'nextcloud',
+    'openproject',
+    'peertube',
+    'pixelfed',
+    'wordpress' 
+]
+
+IMAGES_NO_BACKUP_REQUIRED = [
+    'redis', 
+    'memcached'
+    ]
+
+DIRNAME = os.path.dirname(__file__)
+
+DATABASES = pandas.read_csv(os.path.join(DIRNAME, "databases.csv"), sep=";")
+REPOSITORY_NAME = os.path.basename(DIRNAME)
+MACHINE_ID = get_machine_id()
+BACKUPS_DIR = '/Backups/'
+VERSIONS_DIR = os.path.join(BACKUPS_DIR, MACHINE_ID, REPOSITORY_NAME)
+BACKUP_TIME = datetime.now().strftime("%Y%m%d%H%M%S")
+VERSION_DIR = create_version_directory()
+
+def get_instance(container):
+    # The function is defined to take one parameter, 'container', 
+    # which is expected to be a string.
+
+    # This line uses regular expressions to split the 'container' string.
+    # 're.split' is a method that divides a string into a list, based on the occurrences of a pattern.
+    instance_name = re.split("(_|-)(database|db|postgres)", container)[0]
+    # The pattern "(_|-)(database|db|postgres)" is explained as follows:
+    #    - "(_|-)": Matches an underscore '_' or a hyphen '-'.
+    #    - "(database|db|postgres)": Matches one of the strings "database", "db", or "postgres".
+    # So, this pattern will match segments like "_database", "-db", "_postgres", etc.
+    # For example, in "central-db", it matches "-db".
+
+    # After splitting, [0] is used to select the first element of the list resulting from the split.
+    # This element is the string portion before the matched pattern.
+    # For "central-db", the split results in ["central", "db"], and [0] selects "central".
+
+    print(f"Extracted instance name: {instance_name}")
+    return instance_name
+
+def backup_database(container, volume_dir, db_type):
+    """Backup database (MariaDB or PostgreSQL) if applicable."""
+    print(f"Starting database backup for {container} using {db_type}...")
+    instance_name = get_instance(container)
+
+    # Filter the DataFrame for the given instance_name
+    database_entries = DATABASES.loc[DATABASES['instance'] == instance_name]
+
+    # Check if there are more than one entries
+    if len(database_entries) > 1:
+        raise BackupException(f"More than one entry found for instance '{instance_name}'")
+
+    # Check if there is no entry
+    if database_entries.empty:
+        raise BackupException(f"No entry found for instance '{instance_name}'")
+
+    # Get the first (and only) entry
+    for database_entry in database_entries.iloc:
+        database_name     = database_entry['database']
+        database_username = database_entry['username']
+        database_password = database_entry['password']
+        backup_destination_dir = os.path.join(volume_dir, "sql")
+        pathlib.Path(backup_destination_dir).mkdir(parents=True, exist_ok=True)
+        backup_destination_file = os.path.join(backup_destination_dir, f"{database_name}.backup.sql")
+        if db_type == 'mariadb':
+            backup_command = f"docker exec {container} /usr/bin/mariadb-dump -u {database_username} -p{database_password} {database_name} > {backup_destination_file}"
+        elif db_type == 'postgres':
+            if database_password:
+                # Include PGPASSWORD in the command when a password is provided
+                backup_command = (
+                    f"PGPASSWORD={database_password} docker exec -i {container} "
+                    f"pg_dump -U {database_username} -d {database_name} "
+                    f"-h localhost > {backup_destination_file}"
+                )
+            else:
+                # Exclude PGPASSWORD and use --no-password when the password is empty
+                backup_command = (
+                    f"docker exec -i {container} pg_dump -U {database_username} "
+                    f"-d {database_name} -h localhost --no-password "
+                    f"> {backup_destination_file}"
+                )
+        execute_shell_command(backup_command)
+        print(f"Database backup for database {container} completed.")
+
+def get_last_backup_dir(volume_name, current_backup_dir):
+    """Get the most recent backup directory for the specified volume."""
+    versions = sorted(os.listdir(VERSIONS_DIR), reverse=True)
+    for version in versions:
+        backup_dir = os.path.join(VERSIONS_DIR, version, volume_name, "files")
+        # Ignore current backup dir
+        if backup_dir != current_backup_dir:
+            if os.path.isdir(backup_dir):
+                return backup_dir
+    print(f"No previous backups available for volume: {volume_name}")
+    return None
+
+def getStoragePath(volume_name):
+    return execute_shell_command(f"docker volume inspect --format '{{{{ .Mountpoint }}}}' {volume_name}")
+
+def backup_volume(volume_name, volume_dir):
+    """Backup files of a volume with incremental backups."""
+    print(f"Starting backup routine for volume: {volume_name}")
+    files_rsync_destination_path = os.path.join(volume_dir, "files")
+    pathlib.Path(files_rsync_destination_path).mkdir(parents=True, exist_ok=True)
+
+    last_backup_dir = get_last_backup_dir(volume_name, files_rsync_destination_path)
+    link_dest_option = f"--link-dest='{last_backup_dir}'" if last_backup_dir else ""
+
+    source_dir = getStoragePath(volume_name)
+    rsync_command = f"rsync -abP --delete --delete-excluded {link_dest_option} {source_dir} {files_rsync_destination_path}"
+    execute_shell_command(rsync_command)
+    print(f"Backup routine for volume: {volume_name} completed.")
+
+def get_image_info(container):
+    return execute_shell_command(f"docker inspect --format '{{{{.Config.Image}}}}' {container}")
+
+def has_image(container,image):
+    """Check if the container is using the image"""
+    image_info = get_image_info(container)
+    return image in image_info[0]
+
+def stop_containers(containers):
+    """Stop a list of containers."""
+    for container in containers:
+        print(f"Stopping container {container}...")
+        execute_shell_command(f"docker stop {container}")
+
+def start_containers(containers):
+    """Start a list of stopped containers."""
+    for container in containers:
+        print(f"Starting container {container}...")
+        execute_shell_command(f"docker start {container}")
+
+def get_container_with_image(containers,image):
+    for container in containers:
+        if has_image(container,image):
+            return container
+    return False
+
+def is_image_whitelisted(container, images):
+    """Check if the container's image is one of the whitelisted images."""
+    image_info = get_image_info(container)
+    container_image = image_info[0]
+
+    for image in images:
+        if image in container_image:
+            return True
+    return False
+
+def is_container_stop_required(containers):
+    """Check if any of the containers are using images that are not whitelisted."""
+    return any(not is_image_whitelisted(container, IMAGES_NO_STOP_REQUIRED) for container in containers)
+
+def create_volume_directory(volume_name):
+    """Create necessary directories for backup."""
+    volume_dir = os.path.join(VERSION_DIR, volume_name)
+    pathlib.Path(volume_dir).mkdir(parents=True, exist_ok=True)
+    return volume_dir
+
+def is_image_ignored(container):
+    """Check if the container's image is one of the ignored images."""
+    for image in IMAGES_NO_BACKUP_REQUIRED:
+        if has_image(container, image):
+            return True
+    return False
+
+def backup_with_containers_paused(volume_name, volume_dir, containers, shutdown):
+    stop_containers(containers)
+    backup_volume(volume_name, volume_dir)
+    
+    # Just restart containers if shutdown is false
+    if not shutdown:
+        start_containers(containers)
+
+def backup_mariadb_or_postgres(container, volume_dir):
+    '''Performs database image specific backup procedures'''
+    for image in ['mariadb','postgres']:
+        if has_image(container, image):
+            backup_database(container, volume_dir, image)
+            return True
+    return False
+
+def default_backup_routine_for_volume(volume_name, containers, shutdown):
+    """Perform backup routine for a given volume."""
+    volume_dir=""
+    for container in containers:
+        
+        # Skip ignored images
+        if is_image_ignored(container):
+            print(f"Ignoring volume '{volume_name}' linked to container '{container}' with ignored image.")
+            continue 
+
+        # Directory which contains files and sqls
+        volume_dir = create_volume_directory(volume_name)
+        
+        # Execute Database backup and exit if successfull
+        if backup_mariadb_or_postgres(container, volume_dir):
+            return
+
+    # Execute backup if image is not ignored
+    if volume_dir:    
+        backup_volume(volume_name, volume_dir)
+        if is_container_stop_required(containers):
+            backup_with_containers_paused(volume_name, volume_dir, containers, shutdown)
+
+def backup_everything(volume_name, containers, shutdown):
+    """Perform file backup routine for a given volume."""
+    volume_dir=create_volume_directory(volume_name)
+    
+    # Execute sql dumps
+    for container in containers:
+        backup_mariadb_or_postgres(container, volume_dir)
+
+    # Execute file backups
+    backup_volume(volume_name, volume_dir)
+    backup_with_containers_paused(volume_name, volume_dir, containers, shutdown)
+
+def main():
+    parser = argparse.ArgumentParser(description='Backup Docker volumes.')
+    parser.add_argument('--everything', action='store_true',
+                        help='Force file backup for all volumes and additional execute database dumps')
+    parser.add_argument('--shutdown', action='store_true',
+                        help='Doesn\'t restart containers after backup')
+    args = parser.parse_args()
+
+    print('Start volume backups...')
+    volume_names = execute_shell_command("docker volume ls --format '{{.Name}}'")
+    
+    for volume_name in volume_names:
+        print(f'Start backup routine for volume: {volume_name}')
+        containers = execute_shell_command(f"docker ps --filter volume=\"{volume_name}\" --format '{{{{.Names}}}}'")
+        if not containers:
+            print('Skipped due to no running containers using this volume.')
+            continue
+        if args.everything:
+            backup_everything(volume_name, containers, args.shutdown)
+        else:    
+            default_backup_routine_for_volume(volume_name, containers, args.shutdown)
+
+    print('Finished volume backups.')
+
+if __name__ == "__main__":
+    main()

database_entry_seeder.py

@@ -0,0 +1,44 @@
+import pandas as pd
+import argparse
+import os
+
+def check_and_add_entry(file_path, instance, database, username, password):
+    # Check if the file exists and is not empty
+    if os.path.exists(file_path) and os.path.getsize(file_path) > 0:
+        # Read the existing CSV file with header
+        df = pd.read_csv(file_path, sep=';')
+    else:
+        # Create a new DataFrame with columns if file does not exist
+        df = pd.DataFrame(columns=['instance', 'database', 'username', 'password'])
+
+    # Check if the entry exists and remove it
+    mask = (df['instance'] == instance) & (df['database'] == database) & (df['username'] == username)
+    if not df[mask].empty:
+        print("Replacing existing entry.")
+        df = df[~mask]
+    else:
+        print("Adding new entry.")
+
+    # Create a new DataFrame for the new entry
+    new_entry = pd.DataFrame([{'instance': instance, 'database': database, 'username': username, 'password': password}])
+
+    # Add (or replace) the entry using concat
+    df = pd.concat([df, new_entry], ignore_index=True)
+
+    # Save the updated CSV file
+    df.to_csv(file_path, sep=';', index=False)
+
+def main():
+    parser = argparse.ArgumentParser(description="Check and replace (or add) a database entry in a CSV file.")
+    parser.add_argument("file_path", help="Path to the CSV file")
+    parser.add_argument("instance", help="Database instance")
+    parser.add_argument("database", help="Database name")
+    parser.add_argument("username", help="Username")
+    parser.add_argument("password", nargs='?', default="", help="Password (optional)")
+
+    args = parser.parse_args()
+
+    check_and_add_entry(args.file_path, args.instance, args.database, args.username, args.password)
+
+if __name__ == "__main__":
+    main()

databases.csv.tpl

@@ -0,0 +1 @@
+database;username;password;container

docker-volume-backup.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-# Just backups volumes of running containers
-# If rsync stucks consider:
-# @see https://stackoverflow.com/questions/20773118/rsync-suddenly-hanging-indefinitely-during-transfers
-#
-backup_time="$(date '+%Y%m%d%H%M%S')";
-backups_folder="/Backups/";
-repository_name="$(cd "$(dirname "$(readlink -f "${0}")")" && basename `git rev-parse --show-toplevel`)";
-machine_id="$(sha256sum /etc/machine-id | head -c 64)";
-backup_repository_folder="$backups_folder$machine_id/$repository_name/";
-for volume_name in $(docker volume ls --format '{{.Name}}');
-do
-  echo "start backup routine: $volume_name";
-  for container_name in $(docker ps -a --filter volume="$volume_name" --format '{{.Names}}');
-  do
-    echo "stop container: $container_name" && docker stop "$container_name"
-    for source_path in $(docker inspect --format "{{ range .Mounts }}{{ if eq .Type \"volume\"}}{{ if eq .Name \"$volume_name\"}}{{ println .Destination }}{{ end }}{{ end }}{{ end }}" "$container_name");
-    do
-      destination_path="$backup_repository_folder""latest/$volume_name";
-      raw_destination_path="$destination_path/raw"
-      prepared_destination_path="$destination_path/prepared"
-      log_path="$backup_repository_folder""log.txt";
-      backup_dir_path="$backup_repository_folder""diffs/$backup_time/$volume_name";
-      raw_backup_dir_path="$backup_dir_path/raw";
-      prepared_backup_dir_path="$backup_dir_path/prepared";
-      if [ -d "$destination_path" ]
-        then
-          echo "backup volume: $volume_name";
-        else
-          echo "first backup volume: $volume_name"
-          mkdir -vp "$raw_destination_path";
-          mkdir -vp "$raw_backup_dir_path";
-          mkdir -vp "$prepared_destination_path";
-          mkdir -vp "$prepared_backup_dir_path";
-      fi
-      docker run --rm --volumes-from "$container_name" -v "$backups_folder:$backups_folder" "kevinveenbirkenbach/alpine-rsync" sh -c "
-      rsync -abP --delete --delete-excluded --log-file=$log_path --backup-dir=$raw_backup_dir_path '$source_path/' $raw_destination_path";
-    done
-    echo "start container: $container_name" && docker start "$container_name";
-  done
-  echo "end backup routine: $volume_name";
-done

docker-volume-recover.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-# @param $1 Volume-Name
-volume_name="$1"
-backup_path="$2"
-docker volume create "$volume_name"
-docker run --rm -v "$volume_name:/recover/" -v "$backup_path:/backup/" "kevinveenbirkenbach/alpine-rsync" sh -c "rsync -avv /backup/ /recover/"

recover-docker-from-local.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Check minimum number of arguments
+if [ $# -lt 3 ]; then
+  echo "ERROR: Not enough arguments. Please provide at least a volume name, backup hash, and version."
+  exit 1
+fi
+
+volume_name="$1"          # Volume-Name
+backup_hash="$2"          # Hashed Machine ID
+version="$3"              # version to backup
+container="${4:-}"        # optional
+mysql_root_password="${5:-}"  # optional
+database="${6:-}"         # optional
+backup_folder="Backups/$backup_hash/backup-docker-to-local/$version/$volume_name"
+backup_files="/$backup_folder/files"
+backup_sql="/$backup_folder/sql/backup.sql"
+
+echo "Inspect volume $volume_name"
+docker volume inspect "$volume_name"
+exit_status_volume_inspect=$?
+
+if [ $exit_status_volume_inspect -eq 0 ]; then
+    echo "Volume $volume_name already exists"
+else
+    echo "Create volume $volume_name"
+    docker volume create "$volume_name"
+    if [ $? -ne 0 ]; then
+        echo "ERROR: Failed to create volume $volume_name"
+        exit 1
+    fi
+fi
+
+if [ -f "$backup_sql" ]; then
+  if [ -n "$container" ] && [ -n "$mysql_root_password" ] && [ -n "$database" ]; then
+    echo "recover mysql dump"
+    cat "$backup_sql" | docker exec -i "$container" mariadb -u root --password="$mysql_root_password" "$database"
+    if [ $? -ne 0 ]; then
+        echo "ERROR: Failed to recover mysql dump"
+        exit 1
+    fi
+    exit 0
+  fi
+  echo "A database backup exists, but a parameter is missing. Files will be recovered instead."
+fi 
+
+if [ -d "$backup_files" ]; then    
+  echo "recover files"
+  docker run --rm -v "$volume_name:/recover/" -v "$backup_files:/backup/" "kevinveenbirkenbach/alpine-rsync" sh -c "rsync -avv --delete /backup/ /recover/"
+  if [ $? -ne 0 ]; then
+    echo "ERROR: Failed to recover files"
+    exit 1
+  fi
+  exit 0
+else
+  echo "ERROR: $backup_files doesn't exist"
+  exit 1
+fi
+
+echo "ERROR: Unhandled case"
+exit 1