Added draft mysql dump

2025-09-09 19:57:11 +02:00 · 2021-08-19 21:02:52 +02:00
6 changed files with 73 additions and 168 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1 +0,0 @@
-databases.csv
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
 # docker-volume-backup
-[![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](./LICENSE.txt) [![Travis CI](https://api.travis-ci.org/kevinveenbirkenbach/docker-volume-backup.svg?branch=main)](https://travis-ci.org/kevinveenbirkenbach/docker-volume-backup)
+[![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](./LICENSE.txt) [![Travis CI](https://travis-ci.org/kevinveenbirkenbach/docker-volume-backup.svg?branch=master)](https://travis-ci.org/kevinveenbirkenbach/docker-volume-backup)

 ## goal
 This script backups all docker-volumes with the help of rsync.
@@ -9,7 +9,7 @@ It is part of the following scheme:
 ![backup scheme](https://www.veen.world/wp-content/uploads/2020/12/server-backup-768x567.jpg)
 Further information you will find [in this blog post](https://www.veen.world/2020/12/26/how-i-backup-dedicated-root-servers/).

-## Backup all volumes
+## Backup
 Execute:

 ```bash
@@ -17,46 +17,42 @@ Execute:
 ```

 ## Recover
-
-### database
-```bash
-  docker exec -i mysql_container mysql -uroot -psecret database < db.sql
-```
-
-### volume
 Execute:

 ```bash
-
-bash ./docker-volume-recover.sh "{{volume_name}}" "$(sha256sum /etc/machine-id | head -c 64)" "{{version_to_recover}}"
-
+./docker-volume-recover.sh {{volume_name}} {{backup_path}}
 ```

-### Database
-
 ## Debug
 To checkout what's going on in the mount container type in the following command:

 ```bash
 docker run -it --entrypoint /bin/sh --rm --volumes-from {{container_name}} -v /Backups/:/Backups/ kevinveenbirkenbach/alpine-rsync
 ```
+## Manual Backup
+rsync -aPvv  '***{{source_path}}***/' ***{{destination_path}}***";

-## Setup
-Install pandas
+## Test
+Delete the volume.
+
+```bash
+docker rm -f container-name
+docker volume rm volume-name
+```
+
+Recover the volume:
+
+```bash
+docker volume create volume-name
+docker run --rm -v volume-name:/recover/ -v ~/backup/:/backup/ "kevinveenbirkenbach/alpine-rsync" sh -c "rsync -avv /backup/ /recover/"
+```
+
+Restart the container.

 ## Optimation
 This setup script is not optimized yet for performance. Please optimized this script for performance if you want to use it in a professional environment.

-## Stucking rsync
- https://stackoverflow.com/questions/20773118/rsync-suddenly-hanging-indefinitely-during-transfers
-
 ## More information
- https://docs.docker.com/storage/volumes/
 - https://blog.ssdnodes.com/blog/docker-backup-volumes/
 - https://www.baculasystems.com/blog/docker-backup-containers/
- https://gist.github.com/spalladino/6d981f7b33f6e0afe6bb
- https://stackoverflow.com/questions/26331651/how-can-i-backup-a-docker-container-with-its-data-volumes
- https://netfuture.ch/2013/08/simple-versioned-timemachine-like-backup-using-rsync/
- https://zwischenzugs.com/2016/08/29/bash-to-python-converter/
- https://en.wikipedia.org/wiki/Incremental_backup#Incremental
- https://unix.stackexchange.com/questions/567837/linux-backup-utility-for-incremental-backups
+- https://hub.docker.com/_/mariadb
--- a/databases.csv.tpl
+++ b/databases.csv.tpl
@@ -1 +0,0 @@
-database;username;password;container
--- a/docker-volume-backup.py
+++ b/docker-volume-backup.py
@@ -1,109 +0,0 @@
-#!/bin/python
-# Backups volumes of running containers
-#
-import subprocess
-import os
-import re
-import pathlib
-import pandas
-from datetime import datetime
-
-
-def bash(command):
-    print(command)
-    process = subprocess.Popen([command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
-    out, err = process.communicate()
-    stdout = out.splitlines()
-    output = []
-    for line in stdout:
-        output.append(line.decode("utf-8"))
-    if process.wait() > bool(0):
-        print(command, out, err)
-        raise Exception("Exitcode is greater then 0")
-    return output
-
-
-def print_bash(command):
-    output = bash(command)
-    print(list_to_string(output))
-    return output
-
-
-def list_to_string(list):
-    return str(' '.join(list))
-
-
-print('start backup routine...')
-
-dirname = os.path.dirname(__file__)
-repository_name = os.path.basename(dirname)
-# identifier of this backups
-machine_id = bash("sha256sum /etc/machine-id")[0][0:64]
-# Folder in which all Backups are stored
-backups_dir = '/Backups/'
-# Folder in which the versions off docker volume backups are stored
-versions_dir = backups_dir + machine_id + "/" + repository_name + "/"
-# Time when the backup started
-backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
-# Folder containing the current version
-version_dir = versions_dir + backup_time + "/"
-
-# Create folder to store version in
-pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
-
-print('start volume backups...')
-print('load connection data...')
-databases = pandas.read_csv(dirname + "/databases.csv", sep=";")
-volume_names = bash("docker volume ls --format '{{.Name}}'")
-for volume_name in volume_names:
-    print('start backup routine for volume: ' + volume_name)
-    containers = bash("docker ps --filter volume=\"" + volume_name + "\" --format '{{.Names}}'")
-    if len(containers) == 0:
-        print('skipped due to no running containers using this volume.')
-    else:
-        container = containers[0]
-        # Folder to which the volumes are copied
-        volume_destination_dir = version_dir + volume_name
-        # Database name
-        database_name = re.split("(_|-)(database|db)", container)[0]
-        # Entries with database login data concerning this container
-        databases_entries = databases.loc[databases['database'] == database_name]
-        # Exception for akaunting due to fast implementation
-        if len(databases_entries) == 1 and container != 'akaunting':
-            print("Backup database...")
-            mysqldump_destination_dir = volume_destination_dir + "/sql"
-            mysqldump_destination_file = mysqldump_destination_dir + "/backup.sql"
-            pathlib.Path(mysqldump_destination_dir).mkdir(parents=True, exist_ok=True)
-            database_entry = databases_entries.iloc[0]
-            database_backup_command = "docker exec " + container + " /usr/bin/mariadb-dump -u " + database_entry["username"] + " -p" + database_entry["password"] + " " + database_entry["database"] + " > " + mysqldump_destination_file
-            print_bash(database_backup_command)
-        print("Backup files...")
-        files_rsync_destination_path = volume_destination_dir + "/files"
-        pathlib.Path(files_rsync_destination_path).mkdir(parents=True, exist_ok=True)
-        versions = os.listdir(versions_dir)
-        versions.sort(reverse=True)
-        if len(versions) > 1:
-            last_version = versions[1]
-            last_version_files_dir = versions_dir + last_version + "/" + volume_name + "/files"
-            if os.path.isdir(last_version_files_dir):
-                link_dest_parameter="--link-dest='" + last_version_files_dir + "' "
-            else:
-                print("No previous version exists in path "+ last_version_files_dir + ".")
-                link_dest_parameter=""
-        else:
-            print("No previous version exists in path "+ last_version_files_dir + ".")
-            link_dest_parameter=""
-        source_dir = "/var/lib/docker/volumes/" + volume_name + "/_data/"
-        rsync_command = "rsync -abP --delete --delete-excluded " + link_dest_parameter + source_dir + " " + files_rsync_destination_path
-        print_bash(rsync_command)
-        print("stop containers...")
-        print("Backup data after container is stopped...")
-        print_bash("docker stop " + list_to_string(containers))
-        print_bash(rsync_command)
-        print("start containers...")
-        print_bash("docker start " + list_to_string(containers))
-    print("end backup routine for volume:" + volume_name)
-print('finished volume backups.')
-print('restart docker service...')
-print_bash("systemctl restart docker")
-print('finished backup routine.')
--- a/docker-volume-backup.sh
+++ b/docker-volume-backup.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+# Just backups volumes of running containers
+# If rsync stucks consider:
+# @see https://stackoverflow.com/questions/20773118/rsync-suddenly-hanging-indefinitely-during-transfers
+#
+backup_time="$(date '+%Y%m%d%H%M%S')";
+backups_folder="/Backups/";
+repository_name="$(cd "$(dirname "$(readlink -f "${0}")")" && basename `git rev-parse --show-toplevel`)";
+machine_id="$(sha256sum /etc/machine-id | head -c 64)";
+backup_repository_folder="$backups_folder$machine_id/$repository_name/";
+for volume_name in $(docker volume ls --format '{{.Name}}');
+do
+  echo "start backup routine: $volume_name";
+  for container_name in $(docker ps -a --filter volume="$volume_name" --format '{{.Names}}');
+  do
+    echo "stop container: $container_name" && docker stop "$container_name"
+    for source_path in $(docker inspect --format "{{ range .Mounts }}{{ if eq .Type \"volume\"}}{{ if eq .Name \"$volume_name\"}}{{ println .Destination }}{{ end }}{{ end }}{{ end }}" "$container_name");
+    do
+      destination_path="$backup_repository_folder""latest/$volume_name";
+      raw_destination_path="$destination_path/raw"
+      prepared_destination_path="$destination_path/prepared"
+      log_path="$backup_repository_folder""log.txt";
+      backup_dir_path="$backup_repository_folder""diffs/$backup_time/$volume_name";
+      raw_backup_dir_path="$backup_dir_path/raw";
+      prepared_backup_dir_path="$backup_dir_path/prepared";
+      if [ -d "$destination_path" ]
+        then
+          echo "backup volume: $volume_name";
+        else
+          echo "first backup volume: $volume_name"
+          mkdir -vp "$raw_destination_path";
+          mkdir -vp "$raw_backup_dir_path";
+          mkdir -vp "$prepared_destination_path";
+          mkdir -vp "$prepared_backup_dir_path";
+      fi
+      docker run --rm --volumes-from "$container_name" -v "$backups_folder:$backups_folder" "kevinveenbirkenbach/alpine-rsync" sh -c "
+      rsync -abP --delete --delete-excluded --log-file=$log_path --backup-dir=$raw_backup_dir_path '$source_path/' $raw_destination_path";
+    done
+    echo "start container: $container_name" && docker start "$container_name";
+    if [ "mariadb" == "$(docker inspect --format='{{.Config.Image}}' $container_name)"]
+      then
+        docker exec some-mariadb sh -c 'exec mysqldump --all-databases -uroot -p"$MARIADB_ROOT_PASSWORD"' > /some/path/on/your/host/all-databases.sql
+      fi
+  done
+  echo "end backup routine: $volume_name";
+done
--- a/docker-volume-recover.sh
+++ b/docker-volume-recover.sh
@@ -1,32 +1,6 @@
 #!/bin/bash
-volume_name="$1"          # Volume-Name
-backup_hash="$2"          # Hashed Machine ID
-version="$3"              # version to backup
-container="$4"            # optional
-mysql_root_password="$5"  # optional
-database="$6"             # optional
-backup_folder="Backups/$backup_hash/docker-volume-backup/$version/$volume_name"
-backup_files="/$backup_folder/files"
-backup_sql="/$backup_folder/sql/backup.sql"
-echo "Inspect volume $volume_name"
-docker volume inspect "$volume_name"
-exit_status_volume_inspect=$?
-if [ $exit_status_volume_inspect -eq 0 ]; then
-    echo "Volume $volume_name allready exists"
-  else
-    echo "Create volume $volume_name"
-    docker volume create "$volume_name"
-fi
-
-if [ -f "$backup_sql" ]; then
-  echo "recover mysql dump"
-  cat $backup_sql | docker exec -i "$container" /usr/bin/mysql -u root --password="$mysql_root_password" $database
-  exit 0
-else
-  if [ -d "$backup_files" ]; then    
-    echo "recover files"
-    docker run --rm -v "$volume_name:/recover/" -v "$backup_files:/backup/" "kevinveenbirkenbach/alpine-rsync" sh -c "rsync -avv --delete /backup/ /recover/"
-  fi
-fi
-echo "ERROR: $backup_files and $backup_sql don't exist"
-exit 1
+# @param $1 Volume-Name
+volume_name="$1"
+backup_path="$2"
+docker volume create "$volume_name"
+docker run --rm -v "$volume_name:/recover/" -v "$backup_path:/backup/" "kevinveenbirkenbach/alpine-rsync" sh -c "rsync -avv /backup/ /recover/"