mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2025-12-27 11:06:35 +00:00
Compare commits
11 Commits
41910aece2
...
stable
| Author | SHA1 | Date | |
|---|---|---|---|
| 3b39a6ef02 | |||
| e0b2e8934e | |||
| bbb2dd1732 | |||
| 159502af5e | |||
| 698d1e7a9e | |||
| f8420c8bea | |||
| 8e1a53e1f9 | |||
| 7b55d59300 | |||
| cf6f4d8326 | |||
| 4af15d9074 | |||
| c30b4865d4 |
91
.github/workflows/ci.yml
vendored
Normal file
91
.github/workflows/ci.yml
vendored
Normal file
@@ -0,0 +1,91 @@
|
||||
name: CI (make tests, stable, publish)
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["**"]
|
||||
tags: ["v*.*.*"] # SemVer tags like v1.2.3
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
contents: write # push/update 'stable' tag
|
||||
packages: write # push to GHCR
|
||||
|
||||
env:
|
||||
IMAGE_NAME: baudolo
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_REPO: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: make test
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Show docker info
|
||||
run: |
|
||||
docker version
|
||||
docker info
|
||||
|
||||
- name: Run all tests via Makefile
|
||||
run: |
|
||||
make test
|
||||
|
||||
- name: Upload E2E artifacts (always)
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: e2e-artifacts
|
||||
path: artifacts
|
||||
if-no-files-found: ignore
|
||||
|
||||
stable_and_publish:
|
||||
name: Mark stable + publish image (SemVer tags only)
|
||||
needs: [test]
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
|
||||
steps:
|
||||
- name: Checkout (full history for tags)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Derive version from tag
|
||||
id: ver
|
||||
run: |
|
||||
TAG="${GITHUB_REF#refs/tags/}" # v1.2.3
|
||||
echo "tag=${TAG}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Mark 'stable' git tag (force update)
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git tag -f stable "${GITHUB_SHA}"
|
||||
git push -f origin stable
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build image (Makefile)
|
||||
run: |
|
||||
make build
|
||||
|
||||
- name: Tag image for registry
|
||||
run: |
|
||||
# local image built by Makefile is: baudolo:local
|
||||
docker tag "${IMAGE_NAME}:local" "${REGISTRY}/${IMAGE_REPO}:${{ steps.ver.outputs.tag }}"
|
||||
docker tag "${IMAGE_NAME}:local" "${REGISTRY}/${IMAGE_REPO}:stable"
|
||||
docker tag "${IMAGE_NAME}:local" "${REGISTRY}/${IMAGE_REPO}:sha-${GITHUB_SHA::12}"
|
||||
|
||||
- name: Push image
|
||||
run: |
|
||||
docker push "${REGISTRY}/${IMAGE_REPO}:${{ steps.ver.outputs.tag }}"
|
||||
docker push "${REGISTRY}/${IMAGE_REPO}:stable"
|
||||
docker push "${REGISTRY}/${IMAGE_REPO}:sha-${GITHUB_SHA::12}"
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@@ -1,2 +1,2 @@
|
||||
databases.csv
|
||||
__pycache__
|
||||
__pycache__
|
||||
artifacts/
|
||||
@@ -1,2 +0,0 @@
|
||||
language: shell
|
||||
script: shellcheck $(find . -type f -name '*.sh')
|
||||
4
CHANGELOG.md
Normal file
4
CHANGELOG.md
Normal file
@@ -0,0 +1,4 @@
|
||||
## [1.0.0] - 2025-12-27
|
||||
|
||||
* Official Release 🥳
|
||||
|
||||
34
Dockerfile
Normal file
34
Dockerfile
Normal file
@@ -0,0 +1,34 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Runtime + build essentials:
|
||||
# - rsync: required for file backup/restore
|
||||
# - ca-certificates: TLS
|
||||
# - docker-cli: needed if you want to control the host Docker engine (via /var/run/docker.sock mount)
|
||||
# - make: to delegate install logic to Makefile
|
||||
#
|
||||
# Notes:
|
||||
# - On Debian slim, the docker client package is typically "docker.io".
|
||||
# - If you only want restore-without-docker, you can drop docker.io later.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
make \
|
||||
rsync \
|
||||
ca-certificates \
|
||||
docker-cli \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Fail fast if docker client is missing
|
||||
RUN command -v docker
|
||||
|
||||
COPY . .
|
||||
|
||||
# All install decisions are handled by the Makefile.
|
||||
RUN make install
|
||||
|
||||
# Sensible defaults (can be overridden at runtime)
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
# Default: show CLI help
|
||||
CMD ["baudolo", "--help"]
|
||||
4
MIRRORS
Normal file
4
MIRRORS
Normal file
@@ -0,0 +1,4 @@
|
||||
git@github.com:kevinveenbirkenbach/backup-docker-to-local.git
|
||||
ssh://git@git.veen.world:2201/kevinveenbirkenbach/backup-docker-to-local.git
|
||||
ssh://git@code.infinito.nexus:2201/kevinveenbirkenbach/backup-docker-to-local.git
|
||||
https://pypi.org/project/baudolo/
|
||||
63
Makefile
63
Makefile
@@ -1,12 +1,57 @@
|
||||
.PHONY: test
|
||||
.PHONY: install build \
|
||||
test-e2e test test-unit test-integration
|
||||
|
||||
test:
|
||||
python -m unittest discover -s tests/unit -p "test_*.py"
|
||||
# Default python if no venv is active
|
||||
PY_DEFAULT ?= python3
|
||||
|
||||
IMAGE_NAME ?= baudolo
|
||||
IMAGE_TAG ?= local
|
||||
IMAGE := $(IMAGE_NAME):$(IMAGE_TAG)
|
||||
|
||||
install:
|
||||
@echo ">> Installation instructions:"
|
||||
@echo " This software can be installed with pkgmgr under the alias 'baudolo':"
|
||||
@echo " pkgmgr install baudolo"
|
||||
@echo ""
|
||||
@echo " 📦 pkgmgr project page:"
|
||||
@echo " https://github.com/kevinveenbirkenbach/package-manager"
|
||||
@set -eu; \
|
||||
PY="$(PY_DEFAULT)"; \
|
||||
if [ -n "$${VIRTUAL_ENV:-}" ] && [ -x "$${VIRTUAL_ENV}/bin/python" ]; then \
|
||||
PY="$${VIRTUAL_ENV}/bin/python"; \
|
||||
fi; \
|
||||
echo ">>> Using python: $$PY"; \
|
||||
"$$PY" -m pip install --upgrade pip; \
|
||||
"$$PY" -m pip install -e .; \
|
||||
command -v baudolo >/dev/null 2>&1 || { \
|
||||
echo "ERROR: baudolo not found on PATH after install"; \
|
||||
exit 2; \
|
||||
}; \
|
||||
baudolo --help >/dev/null 2>&1 || true
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Build the baudolo Docker image
|
||||
# ------------------------------------------------------------
|
||||
build:
|
||||
@echo ">> Building Docker image $(IMAGE)"
|
||||
docker build -t $(IMAGE) .
|
||||
|
||||
clean:
|
||||
git clean -fdX .
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Run E2E tests inside the container (Docker socket required)
|
||||
# ------------------------------------------------------------
|
||||
# E2E via isolated Docker-in-Docker (DinD)
|
||||
# - depends on local image build
|
||||
# - starts a DinD daemon container on a dedicated network
|
||||
# - loads the freshly built image into DinD
|
||||
# - runs the unittest suite inside a container that talks to DinD via DOCKER_HOST
|
||||
test-e2e: clean build
|
||||
@bash scripts/test-e2e.sh
|
||||
|
||||
test: test-unit test-integration test-e2e
|
||||
|
||||
test-unit: clean build
|
||||
@echo ">> Running unit tests"
|
||||
@docker run --rm -t $(IMAGE) \
|
||||
sh -lc 'python -m unittest discover -t . -s tests/unit -p "test_*.py" -v'
|
||||
|
||||
test-integration: clean build
|
||||
@echo ">> Running integration tests"
|
||||
@docker run --rm -t $(IMAGE) \
|
||||
sh -lc 'python -m unittest discover -t . -s tests/integration -p "test_*.py" -v'
|
||||
196
README.md
196
README.md
@@ -1,80 +1,196 @@
|
||||
# Backup Docker Volumes to Local (baudolo) 📦🔄
|
||||
[](https://github.com/sponsors/kevinveenbirkenbach) [](https://www.patreon.com/c/kevinveenbirkenbach) [](https://buymeacoffee.com/kevinveenbirkenbach) [](https://s.veen.world/paypaldonate)
|
||||
# baudolo – Deterministic Backup & Restore for Docker Volumes 📦🔄
|
||||
[](https://github.com/sponsors/kevinveenbirkenbach) [](https://www.patreon.com/c/kevinveenbirkenbach) [](https://buymeacoffee.com/kevinveenbirkenbach) [](https://s.veen.world/paypaldonate) [](https://www.gnu.org/licenses/agpl-3.0) [](https://www.docker.com) [](https://www.python.org) [](https://github.com/kevinveenbirkenbach/backup-docker-to-local/stargazers)
|
||||
|
||||
|
||||
**Backup Docker Volumes to Local** is a set of Python and shell scripts that enable you to perform incremental backups of all your Docker volumes using rsync. It is designed to integrate seamlessly with [Kevin's Package Manager](https://github.com/kevinveenbirkenbach/package-manager) under the alias **baudolo**, making it easy to install and manage. The tool supports both file and database recoveries with a clear, automated backup scheme.
|
||||
`baudolo` is a backup and restore system for Docker volumes with
|
||||
**mandatory file backups** and **explicit, deterministic database dumps**.
|
||||
It is designed for environments with many Docker services where:
|
||||
- file-level backups must always exist
|
||||
- database dumps must be intentional, predictable, and auditable
|
||||
|
||||
[](https://www.gnu.org/licenses/agpl-3.0) [](https://www.docker.com) [](https://www.python.org) [](https://github.com/kevinveenbirkenbach/backup-docker-to-local/stargazers)
|
||||
## ✨ Key Features
|
||||
|
||||
## 🎯 Goal
|
||||
- 📦 Incremental Docker volume backups using `rsync --link-dest`
|
||||
- 🗄 Optional SQL dumps for:
|
||||
- PostgreSQL
|
||||
- MariaDB / MySQL
|
||||
- 🌱 Explicit database definition for SQL backups (no auto-discovery)
|
||||
- 🧾 Backup integrity stamping via `dirval` (Python API)
|
||||
- ⏸ Automatic container stop/start when required for consistency
|
||||
- 🚫 Whitelisting of containers that do not require stopping
|
||||
- ♻️ Modular, maintainable Python architecture
|
||||
|
||||
This project automates the backup of Docker volumes using incremental backups (rsync) and supports recovering both files and database dumps (MariaDB/PostgreSQL). A robust directory stamping mechanism ensures data integrity, and the tool also handles restarting Docker Compose services when necessary.
|
||||
|
||||
## 🚀 Features
|
||||
## 🧠 Core Concept (Important!)
|
||||
|
||||
- **Incremental Backups:** Uses rsync with `--link-dest` for efficient, versioned backups.
|
||||
- **Database Backup Support:** Backs up MariaDB and PostgreSQL databases from running containers.
|
||||
- **Volume Recovery:** Provides scripts to recover volumes and databases from backups.
|
||||
- **Docker Compose Integration:** Option to automatically restart Docker Compose services after backup.
|
||||
- **Flexible Configuration:** Easily integrated with your Docker environment with minimal setup.
|
||||
- **Comprehensive Logging:** Detailed command output and error handling for safe operations.
|
||||
`baudolo` **separates file backups from database dumps**.
|
||||
|
||||
## 🛠 Requirements
|
||||
- **Docker volumes are always backed up at file level**
|
||||
- **SQL dumps are created only for explicitly defined databases**
|
||||
|
||||
- **Linux Operating System** (with Docker installed) 🐧
|
||||
- **Python 3.x** 🐍
|
||||
- **Docker & Docker Compose** 🔧
|
||||
- **rsync** installed on your system
|
||||
This results in the following behavior:
|
||||
|
||||
## 📥 Installation
|
||||
| Database defined | File backup | SQL dump |
|
||||
|------------------|-------------|----------|
|
||||
| No | ✔ yes | ✘ no |
|
||||
| Yes | ✔ yes | ✔ yes |
|
||||
|
||||
You can install **Backup Docker Volumes to Local** easily via [Kevin's Package Manager](https://github.com/kevinveenbirkenbach/package-manager) using the alias **baudolo**:
|
||||
## 📁 Backup Layout
|
||||
|
||||
```bash
|
||||
pkgmgr install baudolo
|
||||
Backups are stored in a deterministic, fully nested structure:
|
||||
|
||||
```text
|
||||
<backups-dir>/
|
||||
└── <machine-hash>/
|
||||
└── <repo-name>/
|
||||
└── <timestamp>/
|
||||
└── <volume-name>/
|
||||
├── files/
|
||||
└── sql/
|
||||
└── <database>.backup.sql
|
||||
```
|
||||
|
||||
Alternatively, clone the repository directly:
|
||||
### Meaning of each level
|
||||
|
||||
* `<machine-hash>`
|
||||
SHA256 hash of `/etc/machine-id` (host separation)
|
||||
|
||||
* `<repo-name>`
|
||||
Logical backup namespace (project / stack)
|
||||
|
||||
* `<timestamp>`
|
||||
Backup generation (`YYYYMMDDHHMMSS`)
|
||||
|
||||
* `<volume-name>`
|
||||
Docker volume name
|
||||
|
||||
* `files/`
|
||||
Incremental file backup (rsync)
|
||||
|
||||
* `sql/`
|
||||
Optional SQL dumps (only for defined databases)
|
||||
|
||||
## 🚀 Installation
|
||||
|
||||
### Local (editable install)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/kevinveenbirkenbach/backup-docker-to-local.git
|
||||
cd backup-docker-to-local
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## 🚀 Usage
|
||||
## 🌱 Database Definition (SQL Backup Scope)
|
||||
|
||||
### Backup All Volumes
|
||||
### How SQL backups are defined
|
||||
|
||||
To backup all Docker volumes, simply run:
|
||||
`baudolo` creates SQL dumps **only** for databases that are **explicitly defined**
|
||||
via configuration (e.g. a databases definition file or seeding step).
|
||||
|
||||
If a database is **not defined**:
|
||||
|
||||
* its Docker volume is still backed up (files)
|
||||
* **no SQL dump is created**
|
||||
|
||||
> No database definition → file backup only
|
||||
> Database definition present → file backup + SQL dump
|
||||
|
||||
### Why explicit definition?
|
||||
|
||||
`baudolo` does **not** inspect running containers to guess databases.
|
||||
|
||||
Databases must be explicitly defined to guarantee:
|
||||
|
||||
* deterministic backups
|
||||
* predictable restore behavior
|
||||
* reproducible environments
|
||||
* zero accidental production data exposure
|
||||
|
||||
### Required database metadata
|
||||
|
||||
Each database definition provides:
|
||||
|
||||
* database instance (container or logical instance)
|
||||
* database name
|
||||
* database user
|
||||
* database password
|
||||
|
||||
This information is used by `baudolo` to execute
|
||||
`pg_dump`, `pg_dumpall`, or `mariadb-dump`.
|
||||
|
||||
## 💾 Running a Backup
|
||||
|
||||
```bash
|
||||
./backup-docker-to-local.sh
|
||||
baudolo \
|
||||
--compose-dir /srv/docker \
|
||||
--databases-csv /etc/baudolo/databases.csv \
|
||||
--database-containers central-postgres central-mariadb \
|
||||
--images-no-stop-required alpine postgres mariadb mysql \
|
||||
--images-no-backup-required redis busybox
|
||||
```
|
||||
|
||||
### Recovery
|
||||
### Common Backup Flags
|
||||
|
||||
#### Recover Volume Files
|
||||
| Flag | Description |
|
||||
| --------------- | ------------------------------------------- |
|
||||
| `--everything` | Always stop containers and re-run rsync |
|
||||
| `--dump-only` | Only create SQL dumps, skip file backups |
|
||||
| `--shutdown` | Do not restart containers after backup |
|
||||
| `--backups-dir` | Backup root directory (default: `/Backups`) |
|
||||
| `--repo-name` | Backup namespace under machine hash |
|
||||
|
||||
## ♻️ Restore Operations
|
||||
|
||||
### Restore Volume Files
|
||||
|
||||
```bash
|
||||
bash ./recover-docker-from-local.sh "{{volume_name}}" "$(sha256sum /etc/machine-id | head -c 64)" "{{version_to_recover}}"
|
||||
baudolo-restore files \
|
||||
my-volume \
|
||||
<machine-hash> \
|
||||
<version> \
|
||||
--backups-dir /Backups \
|
||||
--repo-name my-repo
|
||||
```
|
||||
|
||||
#### Recover Database
|
||||
|
||||
For example, to recover a MySQL/MariaDB database:
|
||||
Restore into a **different target volume**:
|
||||
|
||||
```bash
|
||||
docker exec -i mysql_container mysql -uroot -psecret database < db.sql
|
||||
baudolo-restore files \
|
||||
target-volume \
|
||||
<machine-hash> \
|
||||
<version> \
|
||||
--source-volume source-volume
|
||||
```
|
||||
|
||||
#### Debug Mode
|
||||
|
||||
To inspect what’s happening inside a container:
|
||||
### Restore PostgreSQL
|
||||
|
||||
```bash
|
||||
docker run -it --entrypoint /bin/sh --rm --volumes-from {{container_name}} -v /Backups/:/Backups/ kevinveenbirkenbach/alpine-rsync
|
||||
baudolo-restore postgres \
|
||||
my-volume \
|
||||
<machine-hash> \
|
||||
<version> \
|
||||
--container postgres \
|
||||
--db-name appdb \
|
||||
--db-password secret \
|
||||
--empty
|
||||
```
|
||||
|
||||
### Restore MariaDB / MySQL
|
||||
|
||||
```bash
|
||||
baudolo-restore mariadb \
|
||||
my-volume \
|
||||
<machine-hash> \
|
||||
<version> \
|
||||
--container mariadb \
|
||||
--db-name shopdb \
|
||||
--db-password secret \
|
||||
--empty
|
||||
```
|
||||
|
||||
> `baudolo` automatically detects whether `mariadb` or `mysql`
|
||||
> is available inside the container
|
||||
|
||||
## 🔍 Backup Scheme
|
||||
|
||||
The backup mechanism uses incremental backups with rsync and stamps directories with a unique hash. For more details on the backup scheme, check out [this blog post](https://blog.veen.world/blog/2020/12/26/how-i-backup-dedicated-root-servers/).
|
||||
|
||||
@@ -1,382 +0,0 @@
|
||||
#!/bin/python
|
||||
# Backups volumes of running containers
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
import re
|
||||
import pathlib
|
||||
import pandas
|
||||
from datetime import datetime
|
||||
import argparse
|
||||
|
||||
class BackupException(Exception):
|
||||
"""Generic exception for backup errors."""
|
||||
pass
|
||||
|
||||
def execute_shell_command(command):
|
||||
"""Execute a shell command and return its output."""
|
||||
print(command)
|
||||
process = subprocess.Popen(
|
||||
[command],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
shell=True
|
||||
)
|
||||
out, err = process.communicate()
|
||||
if process.returncode != 0:
|
||||
raise BackupException(
|
||||
f"Error in command: {command}\n"
|
||||
f"Output: {out}\nError: {err}\n"
|
||||
f"Exit code: {process.returncode}"
|
||||
)
|
||||
return [line.decode("utf-8") for line in out.splitlines()]
|
||||
|
||||
def create_version_directory():
|
||||
"""Create necessary directories for backup."""
|
||||
version_dir = os.path.join(VERSIONS_DIR, BACKUP_TIME)
|
||||
pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
|
||||
return version_dir
|
||||
|
||||
def get_machine_id():
|
||||
"""Get the machine identifier."""
|
||||
return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
|
||||
|
||||
### GLOBAL CONFIGURATION ###
|
||||
|
||||
# Container names treated as special instances for database backups
|
||||
DATABASE_CONTAINERS = ['central-mariadb', 'central-postgres']
|
||||
|
||||
# Images which do not require container stop for file backups
|
||||
IMAGES_NO_STOP_REQUIRED = []
|
||||
|
||||
# Images to skip entirely
|
||||
IMAGES_NO_BACKUP_REQUIRED = []
|
||||
|
||||
# Compose dirs requiring hard restart
|
||||
DOCKER_COMPOSE_HARD_RESTART_REQUIRED = ['mailu']
|
||||
|
||||
# DEFINE CONSTANTS
|
||||
DIRNAME = os.path.dirname(__file__)
|
||||
SCRIPTS_DIRECTORY = pathlib.Path(os.path.realpath(__file__)).parent.parent
|
||||
DATABASES = pandas.read_csv(os.path.join(DIRNAME, "databases.csv"), sep=";")
|
||||
REPOSITORY_NAME = os.path.basename(DIRNAME)
|
||||
MACHINE_ID = get_machine_id()
|
||||
BACKUPS_DIR = '/Backups/'
|
||||
VERSIONS_DIR = os.path.join(BACKUPS_DIR, MACHINE_ID, REPOSITORY_NAME)
|
||||
BACKUP_TIME = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
VERSION_DIR = create_version_directory()
|
||||
|
||||
def get_instance(container):
|
||||
"""Extract the database instance name based on container name."""
|
||||
if container in DATABASE_CONTAINERS:
|
||||
instance_name = container
|
||||
else:
|
||||
instance_name = re.split("(_|-)(database|db|postgres)", container)[0]
|
||||
print(f"Extracted instance name: {instance_name}")
|
||||
return instance_name
|
||||
|
||||
def stamp_directory():
|
||||
"""Stamp a directory using directory-validator."""
|
||||
stamp_command = f"dirval {VERSION_DIR} --stamp"
|
||||
try:
|
||||
execute_shell_command(stamp_command)
|
||||
print(f"Successfully stamped directory: {VERSION_DIR}")
|
||||
except BackupException as e:
|
||||
print(f"Error running 'dirval' for {VERSION_DIR}: {e}")
|
||||
exit(1)
|
||||
|
||||
def backup_database(container, volume_dir, db_type):
|
||||
"""Backup database (MariaDB or PostgreSQL) if applicable."""
|
||||
print(f"Starting database backup for {container} using {db_type}...")
|
||||
instance_name = get_instance(container)
|
||||
database_entries = DATABASES.loc[DATABASES['instance'] == instance_name]
|
||||
if database_entries.empty:
|
||||
raise BackupException(f"No entry found for instance '{instance_name}'")
|
||||
for database_entry in database_entries.iloc:
|
||||
database_name = database_entry['database']
|
||||
database_username = database_entry['username']
|
||||
database_password = database_entry['password']
|
||||
backup_destination_dir = os.path.join(volume_dir, "sql")
|
||||
pathlib.Path(backup_destination_dir).mkdir(parents=True, exist_ok=True)
|
||||
backup_destination_file = os.path.join(
|
||||
backup_destination_dir,
|
||||
f"{database_name}.backup.sql"
|
||||
)
|
||||
if db_type == 'mariadb':
|
||||
cmd = (
|
||||
f"docker exec {container} "
|
||||
f"/usr/bin/mariadb-dump -u {database_username} "
|
||||
f"-p{database_password} {database_name} > {backup_destination_file}"
|
||||
)
|
||||
execute_shell_command(cmd)
|
||||
if db_type == 'postgres':
|
||||
cluster_file = os.path.join(
|
||||
backup_destination_dir,
|
||||
f"{instance_name}.cluster.backup.sql"
|
||||
)
|
||||
if not database_name:
|
||||
fallback_pg_dumpall(
|
||||
container,
|
||||
database_username,
|
||||
database_password,
|
||||
cluster_file
|
||||
)
|
||||
return
|
||||
try:
|
||||
if database_password:
|
||||
cmd = (
|
||||
f"PGPASSWORD={database_password} docker exec -i {container} "
|
||||
f"pg_dump -U {database_username} -d {database_name} "
|
||||
f"-h localhost > {backup_destination_file}"
|
||||
)
|
||||
else:
|
||||
cmd = (
|
||||
f"docker exec -i {container} pg_dump -U {database_username} "
|
||||
f"-d {database_name} -h localhost --no-password "
|
||||
f"> {backup_destination_file}"
|
||||
)
|
||||
execute_shell_command(cmd)
|
||||
except BackupException as e:
|
||||
print(f"pg_dump failed: {e}")
|
||||
print(f"Falling back to pg_dumpall for instance '{instance_name}'")
|
||||
fallback_pg_dumpall(
|
||||
container,
|
||||
database_username,
|
||||
database_password,
|
||||
cluster_file
|
||||
)
|
||||
print(f"Database backup for database {container} completed.")
|
||||
|
||||
def get_last_backup_dir(volume_name, current_backup_dir):
|
||||
"""Get the most recent backup directory for the specified volume."""
|
||||
versions = sorted(os.listdir(VERSIONS_DIR), reverse=True)
|
||||
for version in versions:
|
||||
backup_dir = os.path.join(
|
||||
VERSIONS_DIR, version, volume_name, "files", ""
|
||||
)
|
||||
if backup_dir != current_backup_dir and os.path.isdir(backup_dir):
|
||||
return backup_dir
|
||||
print(f"No previous backups available for volume: {volume_name}")
|
||||
return None
|
||||
|
||||
def getStoragePath(volume_name):
|
||||
path = execute_shell_command(
|
||||
f"docker volume inspect --format '{{{{ .Mountpoint }}}}' {volume_name}"
|
||||
)[0]
|
||||
return f"{path}/"
|
||||
|
||||
def getFileRsyncDestinationPath(volume_dir):
|
||||
path = os.path.join(volume_dir, "files")
|
||||
return f"{path}/"
|
||||
|
||||
def fallback_pg_dumpall(container, username, password, backup_destination_file):
|
||||
"""Fallback function to run pg_dumpall if pg_dump fails or no DB is defined."""
|
||||
print(f"Running pg_dumpall for container '{container}'...")
|
||||
cmd = (
|
||||
f"PGPASSWORD={password} docker exec -i {container} "
|
||||
f"pg_dumpall -U {username} -h localhost > {backup_destination_file}"
|
||||
)
|
||||
execute_shell_command(cmd)
|
||||
|
||||
def backup_volume(volume_name, volume_dir):
|
||||
"""Perform incremental file backup of a Docker volume."""
|
||||
try:
|
||||
print(f"Starting backup routine for volume: {volume_name}")
|
||||
dest = getFileRsyncDestinationPath(volume_dir)
|
||||
pathlib.Path(dest).mkdir(parents=True, exist_ok=True)
|
||||
last = get_last_backup_dir(volume_name, dest)
|
||||
link_dest = f"--link-dest='{last}'" if last else ""
|
||||
source = getStoragePath(volume_name)
|
||||
cmd = (
|
||||
f"rsync -abP --delete --delete-excluded "
|
||||
f"{link_dest} {source} {dest}"
|
||||
)
|
||||
execute_shell_command(cmd)
|
||||
except BackupException as e:
|
||||
if "file has vanished" in str(e):
|
||||
print("Warning: Some files vanished before transfer. Continuing.")
|
||||
else:
|
||||
raise
|
||||
print(f"Backup routine for volume: {volume_name} completed.")
|
||||
|
||||
def get_image_info(container):
|
||||
return execute_shell_command(
|
||||
f"docker inspect --format '{{{{.Config.Image}}}}' {container}"
|
||||
)
|
||||
|
||||
def has_image(container, image):
|
||||
"""Check if the container is using the image"""
|
||||
info = get_image_info(container)[0]
|
||||
return image in info
|
||||
|
||||
def change_containers_status(containers, status):
|
||||
"""Stop or start a list of containers."""
|
||||
if containers:
|
||||
names = ' '.join(containers)
|
||||
print(f"{status.capitalize()} containers: {names}...")
|
||||
execute_shell_command(f"docker {status} {names}")
|
||||
else:
|
||||
print(f"No containers to {status}.")
|
||||
|
||||
def is_image_whitelisted(container, images):
|
||||
"""
|
||||
Return True if the container's image matches any of the whitelist patterns.
|
||||
Also prints out the image name and the match result.
|
||||
"""
|
||||
# fetch the image (e.g. "nextcloud:23-fpm-alpine")
|
||||
info = get_image_info(container)[0]
|
||||
|
||||
# check against each pattern
|
||||
whitelisted = any(pattern in info for pattern in images)
|
||||
|
||||
# log the result
|
||||
print(f"Container {container!r} → image {info!r} → whitelisted? {whitelisted}", flush=True)
|
||||
|
||||
return whitelisted
|
||||
|
||||
def is_container_stop_required(containers):
|
||||
"""
|
||||
Check if any of the containers are using images that are not whitelisted.
|
||||
If so, print them out and return True; otherwise return False.
|
||||
"""
|
||||
# Find all containers whose image isn’t on the whitelist
|
||||
not_whitelisted = [
|
||||
c for c in containers
|
||||
if not is_image_whitelisted(c, IMAGES_NO_STOP_REQUIRED)
|
||||
]
|
||||
|
||||
if not_whitelisted:
|
||||
print(f"Containers requiring stop because they are not whitelisted: {', '.join(not_whitelisted)}")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def create_volume_directory(volume_name):
|
||||
"""Create necessary directories for backup."""
|
||||
path = os.path.join(VERSION_DIR, volume_name)
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
def is_image_ignored(container):
|
||||
"""Check if the container's image is one of the ignored images."""
|
||||
return any(has_image(container, img) for img in IMAGES_NO_BACKUP_REQUIRED)
|
||||
|
||||
def backup_with_containers_paused(volume_name, volume_dir, containers, shutdown):
|
||||
change_containers_status(containers, 'stop')
|
||||
backup_volume(volume_name, volume_dir)
|
||||
if not shutdown:
|
||||
change_containers_status(containers, 'start')
|
||||
|
||||
def backup_mariadb_or_postgres(container, volume_dir):
|
||||
"""Performs database image specific backup procedures"""
|
||||
for img in ['mariadb', 'postgres']:
|
||||
if has_image(container, img):
|
||||
backup_database(container, volume_dir, img)
|
||||
return True
|
||||
return False
|
||||
|
||||
def default_backup_routine_for_volume(volume_name, containers, shutdown):
|
||||
"""Perform backup routine for a given volume."""
|
||||
vol_dir = ""
|
||||
for c in containers:
|
||||
if is_image_ignored(c):
|
||||
print(f"Ignoring volume '{volume_name}' linked to container '{c}'.")
|
||||
continue
|
||||
vol_dir = create_volume_directory(volume_name)
|
||||
if backup_mariadb_or_postgres(c, vol_dir):
|
||||
return
|
||||
if vol_dir:
|
||||
backup_volume(volume_name, vol_dir)
|
||||
if is_container_stop_required(containers):
|
||||
backup_with_containers_paused(volume_name, vol_dir, containers, shutdown)
|
||||
|
||||
def backup_everything(volume_name, containers, shutdown):
|
||||
"""Perform file backup routine for a given volume."""
|
||||
vol_dir = create_volume_directory(volume_name)
|
||||
for c in containers:
|
||||
backup_mariadb_or_postgres(c, vol_dir)
|
||||
backup_volume(volume_name, vol_dir)
|
||||
backup_with_containers_paused(volume_name, vol_dir, containers, shutdown)
|
||||
|
||||
def hard_restart_docker_services(dir_path):
|
||||
"""Perform a hard restart of docker-compose services in the given directory."""
|
||||
try:
|
||||
print(f"Performing hard restart for docker-compose services in: {dir_path}")
|
||||
subprocess.run(["docker-compose", "down"], cwd=dir_path, check=True)
|
||||
subprocess.run(["docker-compose", "up", "-d"], cwd=dir_path, check=True)
|
||||
print(f"Hard restart completed successfully in: {dir_path}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Error during hard restart in {dir_path}: {e}")
|
||||
exit(2)
|
||||
|
||||
def handle_docker_compose_services(parent_directory):
|
||||
"""Iterate through directories and restart or hard restart services as needed."""
|
||||
for entry in os.scandir(parent_directory):
|
||||
if entry.is_dir():
|
||||
dir_path = entry.path
|
||||
name = os.path.basename(dir_path)
|
||||
print(f"Checking directory: {dir_path}")
|
||||
compose_file = os.path.join(dir_path, "docker-compose.yml")
|
||||
if os.path.isfile(compose_file):
|
||||
print(f"Found docker-compose.yml in {dir_path}.")
|
||||
if name in DOCKER_COMPOSE_HARD_RESTART_REQUIRED:
|
||||
print(f"Directory {name} detected. Performing hard restart...")
|
||||
hard_restart_docker_services(dir_path)
|
||||
else:
|
||||
print(f"No restart required for services in {dir_path}...")
|
||||
else:
|
||||
print(f"No docker-compose.yml found in {dir_path}. Skipping.")
|
||||
|
||||
def main():
|
||||
global DATABASE_CONTAINERS, IMAGES_NO_STOP_REQUIRED
|
||||
parser = argparse.ArgumentParser(description='Backup Docker volumes.')
|
||||
parser.add_argument('--everything', action='store_true',
|
||||
help='Force file backup for all volumes and additional execute database dumps')
|
||||
parser.add_argument('--shutdown', action='store_true',
|
||||
help='Doesn\'t restart containers after backup')
|
||||
parser.add_argument('--compose-dir', type=str, required=True,
|
||||
help='Path to the parent directory containing docker-compose setups')
|
||||
parser.add_argument(
|
||||
'--database-containers',
|
||||
nargs='+',
|
||||
required=True,
|
||||
help='List of container names treated as special instances for database backups'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--images-no-stop-required',
|
||||
nargs='+',
|
||||
required=True,
|
||||
help='List of image names for which containers should not be stopped during file backup'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--images-no-backup-required',
|
||||
nargs='+',
|
||||
help='List of image names for which no backup should be performed (optional)'
|
||||
)
|
||||
args = parser.parse_args()
|
||||
DATABASE_CONTAINERS = args.database_containers
|
||||
IMAGES_NO_STOP_REQUIRED = args.images_no_stop_required
|
||||
if args.images_no_backup_required is not None:
|
||||
global IMAGES_NO_BACKUP_REQUIRED
|
||||
IMAGES_NO_BACKUP_REQUIRED = args.images_no_backup_required
|
||||
|
||||
print('💾 Start volume backups...', flush=True)
|
||||
volume_names = execute_shell_command("docker volume ls --format '{{.Name}}'")
|
||||
for volume_name in volume_names:
|
||||
print(f'Start backup routine for volume: {volume_name}')
|
||||
containers = execute_shell_command(
|
||||
f"docker ps --filter volume=\"{volume_name}\" --format '{{{{.Names}}}}'"
|
||||
)
|
||||
if args.everything:
|
||||
backup_everything(volume_name, containers, args.shutdown)
|
||||
else:
|
||||
default_backup_routine_for_volume(volume_name, containers, args.shutdown)
|
||||
|
||||
stamp_directory()
|
||||
print('Finished volume backups.')
|
||||
|
||||
print('Handling Docker Compose services...')
|
||||
handle_docker_compose_services(args.compose_dir)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1 +0,0 @@
|
||||
instance;database;username;password
|
||||
29
pyproject.toml
Normal file
29
pyproject.toml
Normal file
@@ -0,0 +1,29 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=69", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "backup-docker-to-local"
|
||||
version = "1.0.0"
|
||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
license = { text = "AGPL-3.0-or-later" }
|
||||
authors = [{ name = "Kevin Veen-Birkenbach" }]
|
||||
|
||||
dependencies = [
|
||||
"pandas",
|
||||
"dirval",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
baudolo = "baudolo.backup.__main__:main"
|
||||
baudolo-restore = "baudolo.restore.__main__:main"
|
||||
baudolo-seed = "baudolo.seed.__main__:main"
|
||||
|
||||
[tool.setuptools]
|
||||
package-dir = { "" = "src" }
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
exclude = ["tests*"]
|
||||
@@ -1,85 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Check minimum number of arguments
|
||||
if [ $# -lt 3 ]; then
|
||||
echo "ERROR: Not enough arguments. Please provide at least a volume name, backup hash, and version."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
volume_name="$1" # Volume-Name
|
||||
backup_hash="$2" # Hashed Machine ID
|
||||
version="$3" # version to recover
|
||||
|
||||
# DATABASE PARAMETERS
|
||||
database_type="$4" # Valid values; mariadb, postgress
|
||||
database_container="$5" # optional
|
||||
database_password="$6" # optional
|
||||
database_name="$7" # optional
|
||||
database_user="$database_name"
|
||||
|
||||
|
||||
backup_folder="Backups/$backup_hash/backup-docker-to-local/$version/$volume_name"
|
||||
backup_files="/$backup_folder/files"
|
||||
backup_sql="/$backup_folder/sql/$database_name.backup.sql"
|
||||
|
||||
# DATABASE RECOVERY
|
||||
|
||||
if [ ! -z "$database_type" ]; then
|
||||
if [ "$database_type" = "postgres" ]; then
|
||||
if [ -n "$database_container" ] && [ -n "$database_password" ] && [ -n "$database_name" ]; then
|
||||
echo "Recover PostgreSQL dump"
|
||||
export PGPASSWORD="$database_password"
|
||||
cat "$backup_sql" | docker exec -i "$database_container" psql -v ON_ERROR_STOP=1 -U "$database_user" -d "$database_name"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "ERROR: Failed to recover PostgreSQL dump"
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
elif [ "$database_type" = "mariadb" ]; then
|
||||
if [ -n "$database_container" ] && [ -n "$database_password" ] && [ -n "$database_name" ]; then
|
||||
echo "recover mysql dump"
|
||||
cat "$backup_sql" | docker exec -i "$database_container" mariadb -u "$database_user" --password="$database_password" "$database_name"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "ERROR: Failed to recover mysql dump"
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
echo "A database backup exists, but a parameter is missing."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# FILE RECOVERY
|
||||
|
||||
echo "Inspect volume $volume_name"
|
||||
docker volume inspect "$volume_name"
|
||||
exit_status_volume_inspect=$?
|
||||
|
||||
if [ $exit_status_volume_inspect -eq 0 ]; then
|
||||
echo "Volume $volume_name already exists"
|
||||
else
|
||||
echo "Create volume $volume_name"
|
||||
docker volume create "$volume_name"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "ERROR: Failed to create volume $volume_name"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -d "$backup_files" ]; then
|
||||
echo "recover files"
|
||||
docker run --rm -v "$volume_name:/recover/" -v "$backup_files:/backup/" "kevinveenbirkenbach/alpine-rsync" sh -c "rsync -avv --delete /backup/ /recover/"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "ERROR: Failed to recover files"
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
else
|
||||
echo "ERROR: $backup_files doesn't exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "ERROR: Unhandled case"
|
||||
exit 1
|
||||
@@ -1,5 +0,0 @@
|
||||
pacman:
|
||||
- lsof
|
||||
- python-pandas
|
||||
pkgmgr:
|
||||
- dirval
|
||||
@@ -1,170 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# @todo Not tested yet. Needs to be tested
|
||||
"""
|
||||
restore_backup.py
|
||||
|
||||
A script to recover Docker volumes and database dumps from local backups.
|
||||
Supports an --empty flag to clear the database objects before import (drops all tables/functions etc.).
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
|
||||
def run_command(cmd, capture_output=False, input=None, **kwargs):
|
||||
"""Run a subprocess command and handle errors."""
|
||||
try:
|
||||
result = subprocess.run(cmd, check=True, capture_output=capture_output, input=input, **kwargs)
|
||||
return result
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"ERROR: Command '{' '.join(cmd)}' failed with exit code {e.returncode}")
|
||||
if e.stdout:
|
||||
print(e.stdout.decode())
|
||||
if e.stderr:
|
||||
print(e.stderr.decode())
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def recover_postgres(container, password, db_name, user, backup_sql, empty=False):
|
||||
print("Recovering PostgreSQL dump...")
|
||||
os.environ['PGPASSWORD'] = password
|
||||
if empty:
|
||||
print("Dropping existing PostgreSQL objects...")
|
||||
# Drop all tables, views, sequences, functions in public schema
|
||||
drop_sql = """
|
||||
DO $$ DECLARE r RECORD;
|
||||
BEGIN
|
||||
FOR r IN (
|
||||
SELECT table_name AS name, 'TABLE' AS type FROM information_schema.tables WHERE table_schema='public'
|
||||
UNION ALL
|
||||
SELECT routine_name AS name, 'FUNCTION' AS type FROM information_schema.routines WHERE specific_schema='public'
|
||||
UNION ALL
|
||||
SELECT sequence_name AS name, 'SEQUENCE' AS type FROM information_schema.sequences WHERE sequence_schema='public'
|
||||
) LOOP
|
||||
-- Use %s for type to avoid quoting the SQL keyword
|
||||
EXECUTE format('DROP %s public.%I CASCADE', r.type, r.name);
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
"""
|
||||
run_command([
|
||||
'docker', 'exec', '-i', container,
|
||||
'psql', '-v', 'ON_ERROR_STOP=1', '-U', user, '-d', db_name
|
||||
], input=drop_sql.encode())
|
||||
print("Existing objects dropped.")
|
||||
print("Importing the dump...")
|
||||
with open(backup_sql, 'rb') as f:
|
||||
run_command([
|
||||
'docker', 'exec', '-i', container,
|
||||
'psql', '-v', 'ON_ERROR_STOP=1', '-U', user, '-d', db_name
|
||||
], stdin=f)
|
||||
print("PostgreSQL recovery complete.")
|
||||
|
||||
|
||||
def recover_mariadb(container, password, db_name, user, backup_sql, empty=False):
|
||||
print("Recovering MariaDB dump...")
|
||||
if empty:
|
||||
print("Dropping existing MariaDB tables...")
|
||||
# Disable foreign key checks
|
||||
run_command([
|
||||
'docker', 'exec', container,
|
||||
'mysql', '-u', user, f"--password={password}", '-e', 'SET FOREIGN_KEY_CHECKS=0;'
|
||||
])
|
||||
# Get all table names
|
||||
result = run_command([
|
||||
'docker', 'exec', container,
|
||||
'mysql', '-u', user, f"--password={password}", '-N', '-e',
|
||||
f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{db_name}';"
|
||||
], capture_output=True)
|
||||
tables = result.stdout.decode().split()
|
||||
for tbl in tables:
|
||||
run_command([
|
||||
'docker', 'exec', container,
|
||||
'mysql', '-u', user, f"--password={password}", '-e',
|
||||
f"DROP TABLE IF EXISTS `{db_name}`.`{tbl}`;"
|
||||
])
|
||||
# Enable foreign key checks
|
||||
run_command([
|
||||
'docker', 'exec', container,
|
||||
'mysql', '-u', user, f"--password={password}", '-e', 'SET FOREIGN_KEY_CHECKS=1;'
|
||||
])
|
||||
print("Existing tables dropped.")
|
||||
print("Importing the dump...")
|
||||
with open(backup_sql, 'rb') as f:
|
||||
run_command([
|
||||
'docker', 'exec', '-i', container,
|
||||
'mariadb', '-u', user, f"--password={password}", db_name
|
||||
], stdin=f)
|
||||
print("MariaDB recovery complete.")
|
||||
|
||||
|
||||
def recover_files(volume_name, backup_files):
|
||||
print(f"Inspecting volume {volume_name}...")
|
||||
inspect = subprocess.run(['docker', 'volume', 'inspect', volume_name], stdout=subprocess.DEVNULL)
|
||||
if inspect.returncode != 0:
|
||||
print(f"Volume {volume_name} does not exist. Creating...")
|
||||
run_command(['docker', 'volume', 'create', volume_name])
|
||||
else:
|
||||
print(f"Volume {volume_name} already exists.")
|
||||
|
||||
if not os.path.isdir(backup_files):
|
||||
print(f"ERROR: Backup files folder '{backup_files}' does not exist.")
|
||||
sys.exit(1)
|
||||
|
||||
print("Recovering files...")
|
||||
run_command([
|
||||
'docker', 'run', '--rm',
|
||||
'-v', f"{volume_name}:/recover/",
|
||||
'-v', f"{backup_files}:/backup/",
|
||||
'kevinveenbirkenbach/alpine-rsync',
|
||||
'sh', '-c', 'rsync -avv --delete /backup/ /recover/'
|
||||
])
|
||||
print("File recovery complete.")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Recover Docker volumes and database dumps from local backups.'
|
||||
)
|
||||
parser.add_argument('volume_name', help='Name of the Docker volume')
|
||||
parser.add_argument('backup_hash', help='Hashed Machine ID')
|
||||
parser.add_argument('version', help='Version to recover')
|
||||
|
||||
parser.add_argument('--db-type', choices=['postgres', 'mariadb'], help='Type of database backup')
|
||||
parser.add_argument('--db-container', help='Docker container name for the database')
|
||||
parser.add_argument('--db-password', help='Password for the database user')
|
||||
parser.add_argument('--db-name', help='Name of the database')
|
||||
parser.add_argument('--empty', action='store_true', help='Drop existing database objects before importing')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
volume = args.volume_name
|
||||
backup_hash = args.backup_hash
|
||||
version = args.version
|
||||
|
||||
backup_folder = os.path.join('Backups', backup_hash, 'backup-docker-to-local', version, volume)
|
||||
backup_files = os.path.join(os.sep, backup_folder, 'files')
|
||||
backup_sql = None
|
||||
if args.db_name:
|
||||
backup_sql = os.path.join(os.sep, backup_folder, 'sql', f"{args.db_name}.backup.sql")
|
||||
|
||||
# Database recovery
|
||||
if args.db_type:
|
||||
if not (args.db_container and args.db_password and args.db_name):
|
||||
print("ERROR: A database backup exists, aber ein Parameter fehlt.")
|
||||
sys.exit(1)
|
||||
|
||||
user = args.db_name
|
||||
if args.db_type == 'postgres':
|
||||
recover_postgres(args.db_container, args.db_password, args.db_name, user, backup_sql, empty=args.empty)
|
||||
else:
|
||||
recover_mariadb(args.db_container, args.db_password, args.db_name, user, backup_sql, empty=args.empty)
|
||||
sys.exit(0)
|
||||
|
||||
# File recovery
|
||||
recover_files(volume, backup_files)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,96 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Restore multiple PostgreSQL databases from .backup.sql files via a Docker container.
|
||||
|
||||
Usage:
|
||||
./restore_databases.py /path/to/backup_dir container_name
|
||||
"""
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
import glob
|
||||
|
||||
def run_command(cmd, stdin=None):
|
||||
"""
|
||||
Run a subprocess command and abort immediately on any failure.
|
||||
:param cmd: list of command parts
|
||||
:param stdin: file-like object to use as stdin
|
||||
"""
|
||||
subprocess.run(cmd, stdin=stdin, check=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Restore Postgres databases from backup SQL files via Docker container."
|
||||
)
|
||||
parser.add_argument(
|
||||
"backup_dir",
|
||||
help="Path to directory containing .backup.sql files"
|
||||
)
|
||||
parser.add_argument(
|
||||
"container",
|
||||
help="Name of the Postgres Docker container"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
backup_dir = args.backup_dir
|
||||
container = args.container
|
||||
|
||||
pattern = os.path.join(backup_dir, "*.backup.sql")
|
||||
sql_files = sorted(glob.glob(pattern))
|
||||
if not sql_files:
|
||||
print(f"No .backup.sql files found in {backup_dir}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
for sqlfile in sql_files:
|
||||
# Extract database name by stripping the full suffix '.backup.sql'
|
||||
filename = os.path.basename(sqlfile)
|
||||
if not filename.endswith('.backup.sql'):
|
||||
continue
|
||||
dbname = filename[:-len('.backup.sql')]
|
||||
print(f"=== Processing {sqlfile} → database: {dbname} ===")
|
||||
|
||||
# Drop the database, forcing disconnect of sessions if necessary
|
||||
run_command([
|
||||
"docker", "exec", "-i", container,
|
||||
"psql", "-U", "postgres", "-c",
|
||||
f"DROP DATABASE IF EXISTS \"{dbname}\" WITH (FORCE);"
|
||||
])
|
||||
|
||||
# Create a fresh database
|
||||
run_command([
|
||||
"docker", "exec", "-i", container,
|
||||
"psql", "-U", "postgres", "-c",
|
||||
f"CREATE DATABASE \"{dbname}\";"
|
||||
])
|
||||
|
||||
# Ensure the ownership role exists
|
||||
print(f"Ensuring role '{dbname}' exists...")
|
||||
run_command([
|
||||
"docker", "exec", "-i", container,
|
||||
"psql", "-U", "postgres", "-c",
|
||||
(
|
||||
"DO $$BEGIN "
|
||||
f"IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '{dbname}') THEN "
|
||||
f"CREATE ROLE \"{dbname}\"; "
|
||||
"END IF; "
|
||||
"END$$;"
|
||||
)
|
||||
])
|
||||
|
||||
# Restore the dump into the database by streaming file (will abort on first error)
|
||||
print(f"Restoring dump into {dbname} (this may take a while)…")
|
||||
with open(sqlfile, 'rb') as infile:
|
||||
run_command([
|
||||
"docker", "exec", "-i", container,
|
||||
"psql", "-U", "postgres", "-d", dbname
|
||||
], stdin=infile)
|
||||
|
||||
print(f"✔ {dbname} restored.")
|
||||
|
||||
print("All databases have been restored.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
234
scripts/test-e2e.sh
Executable file
234
scripts/test-e2e.sh
Executable file
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# E2E runner using Docker-in-Docker (DinD) with debug-on-failure
|
||||
#
|
||||
# Debug toggles:
|
||||
# E2E_KEEP_ON_FAIL=1 -> keep DinD + volumes + network if tests fail
|
||||
# E2E_KEEP_VOLUMES=1 -> keep volumes even on success/cleanup
|
||||
# E2E_DEBUG_SHELL=1 -> open an interactive shell in the test container instead of running tests
|
||||
# E2E_ARTIFACTS_DIR=./artifacts
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
NET="${E2E_NET:-baudolo-e2e-net}"
|
||||
DIND="${E2E_DIND_NAME:-baudolo-e2e-dind}"
|
||||
DIND_VOL="${E2E_DIND_VOL:-baudolo-e2e-dind-data}"
|
||||
E2E_TMP_VOL="${E2E_TMP_VOL:-baudolo-e2e-tmp}"
|
||||
|
||||
DIND_HOST="${E2E_DIND_HOST:-tcp://127.0.0.1:2375}"
|
||||
DIND_HOST_IN_NET="${E2E_DIND_HOST_IN_NET:-tcp://${DIND}:2375}"
|
||||
|
||||
IMG="${E2E_IMAGE:-baudolo:local}"
|
||||
RSYNC_IMG="${E2E_RSYNC_IMAGE:-ghcr.io/kevinveenbirkenbach/alpine-rsync}"
|
||||
|
||||
READY_TIMEOUT_SECONDS="${E2E_READY_TIMEOUT_SECONDS:-120}"
|
||||
ARTIFACTS_DIR="${E2E_ARTIFACTS_DIR:-./artifacts}"
|
||||
|
||||
KEEP_ON_FAIL="${E2E_KEEP_ON_FAIL:-0}"
|
||||
KEEP_VOLUMES="${E2E_KEEP_VOLUMES:-0}"
|
||||
DEBUG_SHELL="${E2E_DEBUG_SHELL:-0}"
|
||||
|
||||
FAILED=0
|
||||
TS="$(date +%Y%m%d%H%M%S)"
|
||||
|
||||
mkdir -p "${ARTIFACTS_DIR}"
|
||||
|
||||
log() { echo ">> $*"; }
|
||||
|
||||
dump_debug() {
|
||||
log "DEBUG: collecting diagnostics into ${ARTIFACTS_DIR}"
|
||||
|
||||
{
|
||||
echo "=== Host docker version ==="
|
||||
docker version || true
|
||||
echo
|
||||
echo "=== Host docker info ==="
|
||||
docker info || true
|
||||
echo
|
||||
echo "=== DinD reachable? (docker -H ${DIND_HOST} version) ==="
|
||||
docker -H "${DIND_HOST}" version || true
|
||||
echo
|
||||
} > "${ARTIFACTS_DIR}/debug-host-${TS}.txt" 2>&1 || true
|
||||
|
||||
# DinD logs
|
||||
docker logs --tail=5000 "${DIND}" > "${ARTIFACTS_DIR}/dind-logs-${TS}.txt" 2>&1 || true
|
||||
|
||||
# DinD state
|
||||
{
|
||||
echo "=== docker -H ps -a ==="
|
||||
docker -H "${DIND_HOST}" ps -a || true
|
||||
echo
|
||||
echo "=== docker -H images ==="
|
||||
docker -H "${DIND_HOST}" images || true
|
||||
echo
|
||||
echo "=== docker -H network ls ==="
|
||||
docker -H "${DIND_HOST}" network ls || true
|
||||
echo
|
||||
echo "=== docker -H volume ls ==="
|
||||
docker -H "${DIND_HOST}" volume ls || true
|
||||
echo
|
||||
echo "=== docker -H system df ==="
|
||||
docker -H "${DIND_HOST}" system df || true
|
||||
} > "${ARTIFACTS_DIR}/debug-dind-${TS}.txt" 2>&1 || true
|
||||
|
||||
# Try to capture recent events (best effort; might be noisy)
|
||||
docker -H "${DIND_HOST}" events --since 10m --until 0s \
|
||||
> "${ARTIFACTS_DIR}/dind-events-${TS}.txt" 2>&1 || true
|
||||
|
||||
# Dump shared /tmp content from the tmp volume:
|
||||
# We create a temporary container that mounts the volume, then tar its content.
|
||||
# (Does not rely on host filesystem paths.)
|
||||
log "DEBUG: archiving shared /tmp (volume ${E2E_TMP_VOL})"
|
||||
docker -H "${DIND_HOST}" run --rm \
|
||||
-v "${E2E_TMP_VOL}:/tmp" \
|
||||
alpine:3.20 \
|
||||
sh -lc 'cd /tmp && tar -czf /out.tar.gz . || true' \
|
||||
>/dev/null 2>&1 || true
|
||||
|
||||
# The above writes inside the container FS, not to host. So do it properly:
|
||||
# Use "docker cp" from a temp container.
|
||||
local tmpc="baudolo-e2e-tmpdump-${TS}"
|
||||
docker -H "${DIND_HOST}" rm -f "${tmpc}" >/dev/null 2>&1 || true
|
||||
docker -H "${DIND_HOST}" create --name "${tmpc}" -v "${E2E_TMP_VOL}:/tmp" alpine:3.20 \
|
||||
sh -lc 'cd /tmp && tar -czf /tmpdump.tar.gz . || true' >/dev/null
|
||||
docker -H "${DIND_HOST}" start -a "${tmpc}" >/dev/null 2>&1 || true
|
||||
docker -H "${DIND_HOST}" cp "${tmpc}:/tmpdump.tar.gz" "${ARTIFACTS_DIR}/e2e-tmp-${TS}.tar.gz" >/dev/null 2>&1 || true
|
||||
docker -H "${DIND_HOST}" rm -f "${tmpc}" >/dev/null 2>&1 || true
|
||||
|
||||
log "DEBUG: artifacts written:"
|
||||
ls -la "${ARTIFACTS_DIR}" | sed 's/^/ /' || true
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [ "${FAILED}" -eq 1 ] && [ "${KEEP_ON_FAIL}" = "1" ]; then
|
||||
log "KEEP_ON_FAIL=1 and failure detected -> skipping cleanup."
|
||||
log "Next steps:"
|
||||
echo " - Inspect DinD logs: docker logs ${DIND} | less"
|
||||
echo " - Use DinD daemon: docker -H ${DIND_HOST} ps -a"
|
||||
echo " - Shared tmp vol: docker -H ${DIND_HOST} run --rm -v ${E2E_TMP_VOL}:/tmp alpine:3.20 ls -la /tmp"
|
||||
echo " - DinD docker root: docker -H ${DIND_HOST} run --rm -v ${DIND_VOL}:/var/lib/docker alpine:3.20 ls -la /var/lib/docker/volumes"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Cleanup: stopping ${DIND} and removing network ${NET}"
|
||||
docker rm -f "${DIND}" >/dev/null 2>&1 || true
|
||||
docker network rm "${NET}" >/dev/null 2>&1 || true
|
||||
|
||||
if [ "${KEEP_VOLUMES}" != "1" ]; then
|
||||
docker volume rm -f "${DIND_VOL}" >/dev/null 2>&1 || true
|
||||
docker volume rm -f "${E2E_TMP_VOL}" >/dev/null 2>&1 || true
|
||||
else
|
||||
log "Keeping volumes (E2E_KEEP_VOLUMES=1): ${DIND_VOL}, ${E2E_TMP_VOL}"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
log "Creating network ${NET} (if missing)"
|
||||
docker network inspect "${NET}" >/dev/null 2>&1 || docker network create "${NET}" >/dev/null
|
||||
|
||||
log "Removing old ${DIND} (if any)"
|
||||
docker rm -f "${DIND}" >/dev/null 2>&1 || true
|
||||
|
||||
log "(Re)creating DinD data volume ${DIND_VOL}"
|
||||
docker volume rm -f "${DIND_VOL}" >/dev/null 2>&1 || true
|
||||
docker volume create "${DIND_VOL}" >/dev/null
|
||||
|
||||
log "(Re)creating shared /tmp volume ${E2E_TMP_VOL}"
|
||||
docker volume rm -f "${E2E_TMP_VOL}" >/dev/null 2>&1 || true
|
||||
docker volume create "${E2E_TMP_VOL}" >/dev/null
|
||||
|
||||
log "Starting Docker-in-Docker daemon ${DIND}"
|
||||
docker run -d --privileged \
|
||||
--name "${DIND}" \
|
||||
--network "${NET}" \
|
||||
-e DOCKER_TLS_CERTDIR="" \
|
||||
-v "${DIND_VOL}:/var/lib/docker" \
|
||||
-v "${E2E_TMP_VOL}:/tmp" \
|
||||
-p 2375:2375 \
|
||||
docker:dind \
|
||||
--host=tcp://0.0.0.0:2375 \
|
||||
--tls=false >/dev/null
|
||||
|
||||
log "Waiting for DinD to be ready..."
|
||||
for i in $(seq 1 "${READY_TIMEOUT_SECONDS}"); do
|
||||
if docker -H "${DIND_HOST}" version >/dev/null 2>&1; then
|
||||
log "DinD is ready."
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
if [ "${i}" -eq "${READY_TIMEOUT_SECONDS}" ]; then
|
||||
echo "ERROR: DinD did not become ready in time"
|
||||
docker logs --tail=200 "${DIND}" || true
|
||||
FAILED=1
|
||||
dump_debug || true
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
log "Pre-pulling helper images in DinD..."
|
||||
log " - Pulling: ${RSYNC_IMG}"
|
||||
docker -H "${DIND_HOST}" pull "${RSYNC_IMG}"
|
||||
|
||||
log "Ensuring alpine exists in DinD (for debug helpers)"
|
||||
docker -H "${DIND_HOST}" pull alpine:3.20 >/dev/null
|
||||
|
||||
log "Loading ${IMG} image into DinD..."
|
||||
docker save "${IMG}" | docker -H "${DIND_HOST}" load >/dev/null
|
||||
|
||||
log "Running E2E tests inside DinD"
|
||||
set +e
|
||||
if [ "${DEBUG_SHELL}" = "1" ]; then
|
||||
log "E2E_DEBUG_SHELL=1 -> opening shell in test container"
|
||||
docker run --rm -it \
|
||||
--network "${NET}" \
|
||||
-e DOCKER_HOST="${DIND_HOST_IN_NET}" \
|
||||
-e E2E_RSYNC_IMAGE="${RSYNC_IMG}" \
|
||||
-v "${DIND_VOL}:/var/lib/docker:ro" \
|
||||
-v "${E2E_TMP_VOL}:/tmp" \
|
||||
"${IMG}" \
|
||||
sh -lc '
|
||||
set -e
|
||||
if [ ! -f /etc/machine-id ]; then
|
||||
mkdir -p /etc
|
||||
cat /proc/sys/kernel/random/uuid > /etc/machine-id
|
||||
fi
|
||||
echo ">> DOCKER_HOST=${DOCKER_HOST}"
|
||||
docker ps -a || true
|
||||
exec sh
|
||||
'
|
||||
rc=$?
|
||||
else
|
||||
docker run --rm \
|
||||
--network "${NET}" \
|
||||
-e DOCKER_HOST="${DIND_HOST_IN_NET}" \
|
||||
-e E2E_RSYNC_IMAGE="${RSYNC_IMG}" \
|
||||
-v "${DIND_VOL}:/var/lib/docker:ro" \
|
||||
-v "${E2E_TMP_VOL}:/tmp" \
|
||||
"${IMG}" \
|
||||
sh -lc '
|
||||
set -euo pipefail
|
||||
set -x
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
export TMPDIR=/tmp TMP=/tmp TEMP=/tmp
|
||||
|
||||
if [ ! -f /etc/machine-id ]; then
|
||||
mkdir -p /etc
|
||||
cat /proc/sys/kernel/random/uuid > /etc/machine-id
|
||||
fi
|
||||
|
||||
python -m unittest discover -t . -s tests/e2e -p "test_*.py" -v -f
|
||||
'
|
||||
rc=$?
|
||||
fi
|
||||
set -e
|
||||
|
||||
if [ "${rc}" -ne 0 ]; then
|
||||
FAILED=1
|
||||
echo "ERROR: E2E tests failed (exit code: ${rc})"
|
||||
dump_debug || true
|
||||
exit "${rc}"
|
||||
fi
|
||||
|
||||
log "E2E tests passed."
|
||||
1
src/baudolo/backup/__init__.py
Normal file
1
src/baudolo/backup/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Baudolo backup package."""
|
||||
9
src/baudolo/backup/__main__.py
Normal file
9
src/baudolo/backup/__main__.py
Normal file
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .app import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
183
src/baudolo/backup/app.py
Normal file
183
src/baudolo/backup/app.py
Normal file
@@ -0,0 +1,183 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import pathlib
|
||||
from datetime import datetime
|
||||
|
||||
import pandas
|
||||
from dirval import create_stamp_file
|
||||
|
||||
from .cli import parse_args
|
||||
from .compose import handle_docker_compose_services
|
||||
from .db import backup_database
|
||||
from .docker import (
|
||||
change_containers_status,
|
||||
containers_using_volume,
|
||||
docker_volume_names,
|
||||
get_image_info,
|
||||
has_image,
|
||||
)
|
||||
from .shell import execute_shell_command
|
||||
from .volume import backup_volume
|
||||
|
||||
|
||||
def get_machine_id() -> str:
|
||||
return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
|
||||
|
||||
|
||||
def stamp_directory(version_dir: str) -> None:
|
||||
"""
|
||||
Use dirval as a Python library to stamp the directory (no CLI dependency).
|
||||
"""
|
||||
create_stamp_file(version_dir)
|
||||
|
||||
|
||||
def create_version_directory(versions_dir: str, backup_time: str) -> str:
|
||||
version_dir = os.path.join(versions_dir, backup_time)
|
||||
pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
|
||||
return version_dir
|
||||
|
||||
|
||||
def create_volume_directory(version_dir: str, volume_name: str) -> str:
|
||||
path = os.path.join(version_dir, volume_name)
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def is_image_ignored(container: str, images_no_backup_required: list[str]) -> bool:
|
||||
if not images_no_backup_required:
|
||||
return False
|
||||
img = get_image_info(container)
|
||||
return any(pat in img for pat in images_no_backup_required)
|
||||
|
||||
|
||||
def volume_is_fully_ignored(containers: list[str], images_no_backup_required: list[str]) -> bool:
|
||||
"""
|
||||
Skip file backup only if all containers linked to the volume are ignored.
|
||||
"""
|
||||
if not containers:
|
||||
return False
|
||||
return all(is_image_ignored(c, images_no_backup_required) for c in containers)
|
||||
|
||||
|
||||
def requires_stop(containers: list[str], images_no_stop_required: list[str]) -> bool:
|
||||
"""
|
||||
Stop is required if ANY container image is NOT in the whitelist patterns.
|
||||
"""
|
||||
for c in containers:
|
||||
img = get_image_info(c)
|
||||
if not any(pat in img for pat in images_no_stop_required):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def backup_mariadb_or_postgres(
|
||||
*,
|
||||
container: str,
|
||||
volume_dir: str,
|
||||
databases_df: "pandas.DataFrame",
|
||||
database_containers: list[str],
|
||||
) -> bool:
|
||||
"""
|
||||
Returns True if the container is a DB container we handled.
|
||||
"""
|
||||
for img in ["mariadb", "postgres"]:
|
||||
if has_image(container, img):
|
||||
backup_database(
|
||||
container=container,
|
||||
volume_dir=volume_dir,
|
||||
db_type=img,
|
||||
databases_df=databases_df,
|
||||
database_containers=database_containers,
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _backup_dumps_for_volume(
|
||||
*,
|
||||
containers: list[str],
|
||||
vol_dir: str,
|
||||
databases_df: "pandas.DataFrame",
|
||||
database_containers: list[str],
|
||||
) -> bool:
|
||||
"""
|
||||
Create DB dumps for any mariadb/postgres containers attached to this volume.
|
||||
Returns True if at least one dump was produced.
|
||||
"""
|
||||
dumped_any = False
|
||||
for c in containers:
|
||||
if backup_mariadb_or_postgres(
|
||||
container=c,
|
||||
volume_dir=vol_dir,
|
||||
databases_df=databases_df,
|
||||
database_containers=database_containers,
|
||||
):
|
||||
dumped_any = True
|
||||
return dumped_any
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
machine_id = get_machine_id()
|
||||
backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
|
||||
versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
|
||||
version_dir = create_version_directory(versions_dir, backup_time)
|
||||
|
||||
databases_df = pandas.read_csv(args.databases_csv, sep=";")
|
||||
|
||||
print("💾 Start volume backups...", flush=True)
|
||||
|
||||
for volume_name in docker_volume_names():
|
||||
print(f"Start backup routine for volume: {volume_name}", flush=True)
|
||||
containers = containers_using_volume(volume_name)
|
||||
|
||||
vol_dir = create_volume_directory(version_dir, volume_name)
|
||||
|
||||
# Old behavior: DB dumps are additional to file backups.
|
||||
_backup_dumps_for_volume(
|
||||
containers=containers,
|
||||
vol_dir=vol_dir,
|
||||
databases_df=databases_df,
|
||||
database_containers=args.database_containers,
|
||||
)
|
||||
|
||||
# dump-only: skip ALL file rsync backups
|
||||
if args.dump_only:
|
||||
continue
|
||||
|
||||
# skip file backup if all linked containers are ignored
|
||||
if volume_is_fully_ignored(containers, args.images_no_backup_required):
|
||||
print(
|
||||
f"Skipping file backup for volume '{volume_name}' (all linked containers are ignored).",
|
||||
flush=True,
|
||||
)
|
||||
continue
|
||||
|
||||
if args.everything:
|
||||
# "everything": always do pre-rsync, then stop + rsync again
|
||||
backup_volume(versions_dir, volume_name, vol_dir)
|
||||
change_containers_status(containers, "stop")
|
||||
backup_volume(versions_dir, volume_name, vol_dir)
|
||||
if not args.shutdown:
|
||||
change_containers_status(containers, "start")
|
||||
continue
|
||||
|
||||
# default: rsync, and if needed stop + rsync
|
||||
backup_volume(versions_dir, volume_name, vol_dir)
|
||||
if requires_stop(containers, args.images_no_stop_required):
|
||||
change_containers_status(containers, "stop")
|
||||
backup_volume(versions_dir, volume_name, vol_dir)
|
||||
if not args.shutdown:
|
||||
change_containers_status(containers, "start")
|
||||
|
||||
# Stamp the backup version directory using dirval (python lib)
|
||||
stamp_directory(version_dir)
|
||||
print("Finished volume backups.", flush=True)
|
||||
|
||||
print("Handling Docker Compose services...", flush=True)
|
||||
handle_docker_compose_services(args.compose_dir, args.docker_compose_hard_restart_required)
|
||||
|
||||
return 0
|
||||
93
src/baudolo/backup/cli.py
Normal file
93
src/baudolo/backup/cli.py
Normal file
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _default_repo_name() -> str:
|
||||
"""
|
||||
Derive the repository name from the folder that contains `src/`.
|
||||
|
||||
Expected layout:
|
||||
<repo-root>/src/baudolo/backup/cli.py
|
||||
|
||||
=> parents[0]=backup, [1]=baudolo, [2]=src, [3]=repo-root
|
||||
"""
|
||||
try:
|
||||
return Path(__file__).resolve().parents[3].name
|
||||
except Exception:
|
||||
return "backup-docker-to-local"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
dirname = os.path.dirname(__file__)
|
||||
default_databases_csv = os.path.join(dirname, "databases.csv")
|
||||
|
||||
p = argparse.ArgumentParser(description="Backup Docker volumes.")
|
||||
|
||||
p.add_argument(
|
||||
"--compose-dir",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Path to the parent directory containing docker-compose setups",
|
||||
)
|
||||
p.add_argument(
|
||||
"--docker-compose-hard-restart-required",
|
||||
nargs="+",
|
||||
default=["mailu"],
|
||||
help="Compose dir names that require 'docker-compose down && up -d' (default: mailu)",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--repo-name",
|
||||
default=_default_repo_name(),
|
||||
help="Backup repo folder name under <backups-dir>/<machine-id>/ (default: git repo folder name)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--databases-csv",
|
||||
default=default_databases_csv,
|
||||
help=f"Path to databases.csv (default: {default_databases_csv})",
|
||||
)
|
||||
p.add_argument(
|
||||
"--backups-dir",
|
||||
default="/Backups",
|
||||
help="Backup root directory (default: /Backups)",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--database-containers",
|
||||
nargs="+",
|
||||
required=True,
|
||||
help="Container names treated as special instances for database backups",
|
||||
)
|
||||
p.add_argument(
|
||||
"--images-no-stop-required",
|
||||
nargs="+",
|
||||
required=True,
|
||||
help="Image name patterns for which containers should not be stopped during file backup",
|
||||
)
|
||||
p.add_argument(
|
||||
"--images-no-backup-required",
|
||||
nargs="+",
|
||||
default=[],
|
||||
help="Image name patterns for which no backup should be performed",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--everything",
|
||||
action="store_true",
|
||||
help="Force file backup for all volumes and also execute database dumps (like old script)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--shutdown",
|
||||
action="store_true",
|
||||
help="Do not restart containers after backup",
|
||||
)
|
||||
p.add_argument(
|
||||
"--dump-only",
|
||||
action="store_true",
|
||||
help="Only create DB dumps (skip ALL file rsync backups)",
|
||||
)
|
||||
|
||||
return p.parse_args()
|
||||
31
src/baudolo/backup/compose.py
Normal file
31
src/baudolo/backup/compose.py
Normal file
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
|
||||
def hard_restart_docker_services(dir_path: str) -> None:
|
||||
print(f"Hard restart docker-compose services in: {dir_path}", flush=True)
|
||||
subprocess.run(["docker-compose", "down"], cwd=dir_path, check=True)
|
||||
subprocess.run(["docker-compose", "up", "-d"], cwd=dir_path, check=True)
|
||||
|
||||
|
||||
def handle_docker_compose_services(parent_directory: str, hard_restart_required: list[str]) -> None:
|
||||
for entry in os.scandir(parent_directory):
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
|
||||
dir_path = entry.path
|
||||
name = os.path.basename(dir_path)
|
||||
compose_file = os.path.join(dir_path, "docker-compose.yml")
|
||||
|
||||
print(f"Checking directory: {dir_path}", flush=True)
|
||||
if not os.path.isfile(compose_file):
|
||||
print("No docker-compose.yml found. Skipping.", flush=True)
|
||||
continue
|
||||
|
||||
if name in hard_restart_required:
|
||||
print(f"{name}: hard restart required.", flush=True)
|
||||
hard_restart_docker_services(dir_path)
|
||||
else:
|
||||
print(f"{name}: no restart required.", flush=True)
|
||||
73
src/baudolo/backup/db.py
Normal file
73
src/baudolo/backup/db.py
Normal file
@@ -0,0 +1,73 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
|
||||
import pandas
|
||||
|
||||
from .shell import BackupException, execute_shell_command
|
||||
|
||||
|
||||
def get_instance(container: str, database_containers: list[str]) -> str:
|
||||
if container in database_containers:
|
||||
return container
|
||||
return re.split(r"(_|-)(database|db|postgres)", container)[0]
|
||||
|
||||
|
||||
def fallback_pg_dumpall(container: str, username: str, password: str, out_file: str) -> None:
|
||||
cmd = (
|
||||
f"PGPASSWORD={password} docker exec -i {container} "
|
||||
f"pg_dumpall -U {username} -h localhost > {out_file}"
|
||||
)
|
||||
execute_shell_command(cmd)
|
||||
|
||||
|
||||
def backup_database(
|
||||
*,
|
||||
container: str,
|
||||
volume_dir: str,
|
||||
db_type: str,
|
||||
databases_df: "pandas.DataFrame",
|
||||
database_containers: list[str],
|
||||
) -> None:
|
||||
instance_name = get_instance(container, database_containers)
|
||||
entries = databases_df.loc[databases_df["instance"] == instance_name]
|
||||
if entries.empty:
|
||||
raise BackupException(f"No entry found for instance '{instance_name}'")
|
||||
|
||||
out_dir = os.path.join(volume_dir, "sql")
|
||||
pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for row in entries.iloc:
|
||||
db_name = row["database"]
|
||||
user = row["username"]
|
||||
password = row["password"]
|
||||
|
||||
dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
|
||||
|
||||
if db_type == "mariadb":
|
||||
cmd = (
|
||||
f"docker exec {container} /usr/bin/mariadb-dump "
|
||||
f"-u {user} -p{password} {db_name} > {dump_file}"
|
||||
)
|
||||
execute_shell_command(cmd)
|
||||
continue
|
||||
|
||||
if db_type == "postgres":
|
||||
cluster_file = os.path.join(out_dir, f"{instance_name}.cluster.backup.sql")
|
||||
if not db_name:
|
||||
fallback_pg_dumpall(container, user, password, cluster_file)
|
||||
return
|
||||
|
||||
try:
|
||||
cmd = (
|
||||
f"PGPASSWORD={password} docker exec -i {container} "
|
||||
f"pg_dump -U {user} -d {db_name} -h localhost > {dump_file}"
|
||||
)
|
||||
execute_shell_command(cmd)
|
||||
except BackupException as e:
|
||||
print(f"pg_dump failed: {e}", flush=True)
|
||||
print(f"Falling back to pg_dumpall for instance '{instance_name}'", flush=True)
|
||||
fallback_pg_dumpall(container, user, password, cluster_file)
|
||||
continue
|
||||
43
src/baudolo/backup/docker.py
Normal file
43
src/baudolo/backup/docker.py
Normal file
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .shell import execute_shell_command
|
||||
|
||||
|
||||
def get_image_info(container: str) -> str:
|
||||
return execute_shell_command(
|
||||
f"docker inspect --format '{{{{.Config.Image}}}}' {container}"
|
||||
)[0]
|
||||
|
||||
|
||||
def has_image(container: str, pattern: str) -> bool:
|
||||
"""Return True if container's image contains the pattern."""
|
||||
return pattern in get_image_info(container)
|
||||
|
||||
|
||||
def docker_volume_names() -> list[str]:
|
||||
return execute_shell_command("docker volume ls --format '{{.Name}}'")
|
||||
|
||||
|
||||
def containers_using_volume(volume_name: str) -> list[str]:
|
||||
return execute_shell_command(
|
||||
f"docker ps --filter volume=\"{volume_name}\" --format '{{{{.Names}}}}'"
|
||||
)
|
||||
|
||||
|
||||
def change_containers_status(containers: list[str], status: str) -> None:
|
||||
"""Stop or start a list of containers."""
|
||||
if not containers:
|
||||
print(f"No containers to {status}.", flush=True)
|
||||
return
|
||||
names = " ".join(containers)
|
||||
print(f"{status.capitalize()} containers: {names}...", flush=True)
|
||||
execute_shell_command(f"docker {status} {names}")
|
||||
|
||||
|
||||
def docker_volume_exists(volume: str) -> bool:
|
||||
# Avoid throwing exceptions for exists checks.
|
||||
try:
|
||||
execute_shell_command(f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK")
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
26
src/baudolo/backup/shell.py
Normal file
26
src/baudolo/backup/shell.py
Normal file
@@ -0,0 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
|
||||
|
||||
class BackupException(Exception):
|
||||
"""Generic exception for backup errors."""
|
||||
|
||||
|
||||
def execute_shell_command(command: str) -> list[str]:
|
||||
"""Execute a shell command and return its output lines."""
|
||||
print(command, flush=True)
|
||||
process = subprocess.Popen(
|
||||
[command],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
shell=True,
|
||||
)
|
||||
out, err = process.communicate()
|
||||
if process.returncode != 0:
|
||||
raise BackupException(
|
||||
f"Error in command: {command}\n"
|
||||
f"Output: {out}\nError: {err}\n"
|
||||
f"Exit code: {process.returncode}"
|
||||
)
|
||||
return [line.decode("utf-8") for line in out.splitlines()]
|
||||
42
src/baudolo/backup/volume.py
Normal file
42
src/baudolo/backup/volume.py
Normal file
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
from .shell import BackupException, execute_shell_command
|
||||
|
||||
|
||||
def get_storage_path(volume_name: str) -> str:
|
||||
path = execute_shell_command(
|
||||
f"docker volume inspect --format '{{{{ .Mountpoint }}}}' {volume_name}"
|
||||
)[0]
|
||||
return f"{path}/"
|
||||
|
||||
|
||||
def get_last_backup_dir(versions_dir: str, volume_name: str, current_backup_dir: str) -> str | None:
|
||||
versions = sorted(os.listdir(versions_dir), reverse=True)
|
||||
for version in versions:
|
||||
candidate = os.path.join(versions_dir, version, volume_name, "files", "")
|
||||
if candidate != current_backup_dir and os.path.isdir(candidate):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def backup_volume(versions_dir: str, volume_name: str, volume_dir: str) -> None:
|
||||
"""Perform incremental file backup of a Docker volume."""
|
||||
dest = os.path.join(volume_dir, "files") + "/"
|
||||
pathlib.Path(dest).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
last = get_last_backup_dir(versions_dir, volume_name, dest)
|
||||
link_dest = f"--link-dest='{last}'" if last else ""
|
||||
source = get_storage_path(volume_name)
|
||||
|
||||
cmd = f"rsync -abP --delete --delete-excluded {link_dest} {source} {dest}"
|
||||
|
||||
try:
|
||||
execute_shell_command(cmd)
|
||||
except BackupException as e:
|
||||
if "file has vanished" in str(e):
|
||||
print("Warning: Some files vanished before transfer. Continuing.", flush=True)
|
||||
else:
|
||||
raise
|
||||
1
src/baudolo/restore/__init__.py
Normal file
1
src/baudolo/restore/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
__all__ = ["main"]
|
||||
144
src/baudolo/restore/__main__.py
Normal file
144
src/baudolo/restore/__main__.py
Normal file
@@ -0,0 +1,144 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from .paths import BackupPaths
|
||||
from .files import restore_volume_files
|
||||
from .db.postgres import restore_postgres_sql
|
||||
from .db.mariadb import restore_mariadb_sql
|
||||
|
||||
|
||||
def _add_common_backup_args(p: argparse.ArgumentParser) -> None:
|
||||
p.add_argument("volume_name", help="Docker volume name (target volume)")
|
||||
p.add_argument("backup_hash", help="Hashed machine id")
|
||||
p.add_argument("version", help="Backup version directory name")
|
||||
|
||||
p.add_argument(
|
||||
"--backups-dir",
|
||||
default="/Backups",
|
||||
help="Backup root directory (default: /Backups)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--repo-name",
|
||||
default="backup-docker-to-local",
|
||||
help="Backup repo folder name under <backups-dir>/<hash>/ (default: backup-docker-to-local)",
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="baudolo-restore",
|
||||
description="Restore docker volume files and DB dumps.",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# files
|
||||
# ------------------------------------------------------------------
|
||||
p_files = sub.add_parser("files", help="Restore files into a docker volume")
|
||||
_add_common_backup_args(p_files)
|
||||
p_files.add_argument(
|
||||
"--rsync-image",
|
||||
default="ghcr.io/kevinveenbirkenbach/alpine-rsync",
|
||||
)
|
||||
p_files.add_argument(
|
||||
"--source-volume",
|
||||
default=None,
|
||||
help=(
|
||||
"Volume name used as backup source path key. "
|
||||
"Defaults to <volume_name> (target volume). "
|
||||
"Use this when restoring from one volume backup into a different target volume."
|
||||
),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# postgres
|
||||
# ------------------------------------------------------------------
|
||||
p_pg = sub.add_parser("postgres", help="Restore a single PostgreSQL database dump")
|
||||
_add_common_backup_args(p_pg)
|
||||
p_pg.add_argument("--container", required=True)
|
||||
p_pg.add_argument("--db-name", required=True)
|
||||
p_pg.add_argument("--db-user", default=None, help="Defaults to db-name if omitted")
|
||||
p_pg.add_argument("--db-password", required=True)
|
||||
p_pg.add_argument("--empty", action="store_true")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# mariadb
|
||||
# ------------------------------------------------------------------
|
||||
p_mdb = sub.add_parser("mariadb", help="Restore a single MariaDB/MySQL-compatible dump")
|
||||
_add_common_backup_args(p_mdb)
|
||||
p_mdb.add_argument("--container", required=True)
|
||||
p_mdb.add_argument("--db-name", required=True)
|
||||
p_mdb.add_argument("--db-user", default=None, help="Defaults to db-name if omitted")
|
||||
p_mdb.add_argument("--db-password", required=True)
|
||||
p_mdb.add_argument("--empty", action="store_true")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
if args.cmd == "files":
|
||||
# target volume = args.volume_name
|
||||
# source volume (backup key) defaults to target volume
|
||||
source_volume = args.source_volume or args.volume_name
|
||||
|
||||
bp_files = BackupPaths(
|
||||
source_volume,
|
||||
args.backup_hash,
|
||||
args.version,
|
||||
repo_name=args.repo_name,
|
||||
backups_dir=args.backups_dir,
|
||||
)
|
||||
|
||||
return restore_volume_files(
|
||||
args.volume_name,
|
||||
bp_files.files_dir(),
|
||||
rsync_image=args.rsync_image,
|
||||
)
|
||||
|
||||
if args.cmd == "postgres":
|
||||
user = args.db_user or args.db_name
|
||||
restore_postgres_sql(
|
||||
container=args.container,
|
||||
db_name=args.db_name,
|
||||
user=user,
|
||||
password=args.db_password,
|
||||
sql_path=BackupPaths(
|
||||
args.volume_name,
|
||||
args.backup_hash,
|
||||
args.version,
|
||||
repo_name=args.repo_name,
|
||||
backups_dir=args.backups_dir,
|
||||
).sql_file(args.db_name),
|
||||
empty=args.empty,
|
||||
)
|
||||
return 0
|
||||
|
||||
if args.cmd == "mariadb":
|
||||
user = args.db_user or args.db_name
|
||||
restore_mariadb_sql(
|
||||
container=args.container,
|
||||
db_name=args.db_name,
|
||||
user=user,
|
||||
password=args.db_password,
|
||||
sql_path=BackupPaths(
|
||||
args.volume_name,
|
||||
args.backup_hash,
|
||||
args.version,
|
||||
repo_name=args.repo_name,
|
||||
backups_dir=args.backups_dir,
|
||||
).sql_file(args.db_name),
|
||||
empty=args.empty,
|
||||
)
|
||||
return 0
|
||||
|
||||
parser.error("Unhandled command")
|
||||
return 2
|
||||
|
||||
except Exception as e:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1
src/baudolo/restore/db/__init__.py
Normal file
1
src/baudolo/restore/db/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Database restore handlers (Postgres, MariaDB/MySQL)."""
|
||||
89
src/baudolo/restore/db/mariadb.py
Normal file
89
src/baudolo/restore/db/mariadb.py
Normal file
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from ..run import docker_exec, docker_exec_sh
|
||||
|
||||
|
||||
def _pick_client(container: str) -> str:
|
||||
"""
|
||||
Prefer 'mariadb', fallback to 'mysql'.
|
||||
Some MariaDB images no longer ship a 'mysql' binary, so we must not assume it exists.
|
||||
"""
|
||||
script = r"""
|
||||
set -eu
|
||||
if command -v mariadb >/dev/null 2>&1; then echo mariadb; exit 0; fi
|
||||
if command -v mysql >/dev/null 2>&1; then echo mysql; exit 0; fi
|
||||
exit 42
|
||||
"""
|
||||
try:
|
||||
out = docker_exec_sh(container, script, capture=True).stdout.decode().strip()
|
||||
if not out:
|
||||
raise RuntimeError("empty client detection output")
|
||||
return out
|
||||
except Exception as e:
|
||||
print("ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr)
|
||||
raise e
|
||||
|
||||
|
||||
def restore_mariadb_sql(
|
||||
*,
|
||||
container: str,
|
||||
db_name: str,
|
||||
user: str,
|
||||
password: str,
|
||||
sql_path: str,
|
||||
empty: bool,
|
||||
) -> None:
|
||||
client = _pick_client(container)
|
||||
|
||||
if not os.path.isfile(sql_path):
|
||||
raise FileNotFoundError(sql_path)
|
||||
|
||||
if empty:
|
||||
# IMPORTANT:
|
||||
# Do NOT hardcode 'mysql' here. Use the detected client.
|
||||
# MariaDB 11 images may not contain the mysql binary at all.
|
||||
docker_exec(
|
||||
container,
|
||||
[client, "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=0;"],
|
||||
)
|
||||
|
||||
result = docker_exec(
|
||||
container,
|
||||
[
|
||||
client,
|
||||
"-u",
|
||||
user,
|
||||
f"--password={password}",
|
||||
"-N",
|
||||
"-e",
|
||||
f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{db_name}';",
|
||||
],
|
||||
capture=True,
|
||||
)
|
||||
tables = result.stdout.decode().split()
|
||||
|
||||
for tbl in tables:
|
||||
docker_exec(
|
||||
container,
|
||||
[
|
||||
client,
|
||||
"-u",
|
||||
user,
|
||||
f"--password={password}",
|
||||
"-e",
|
||||
f"DROP TABLE IF EXISTS `{db_name}`.`{tbl}`;",
|
||||
],
|
||||
)
|
||||
|
||||
docker_exec(
|
||||
container,
|
||||
[client, "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=1;"],
|
||||
)
|
||||
|
||||
with open(sql_path, "rb") as f:
|
||||
docker_exec(container, [client, "-u", user, f"--password={password}", db_name], stdin=f)
|
||||
|
||||
print(f"MariaDB/MySQL restore complete for db '{db_name}'.")
|
||||
53
src/baudolo/restore/db/postgres.py
Normal file
53
src/baudolo/restore/db/postgres.py
Normal file
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from ..run import docker_exec
|
||||
|
||||
|
||||
def restore_postgres_sql(
|
||||
*,
|
||||
container: str,
|
||||
db_name: str,
|
||||
user: str,
|
||||
password: str,
|
||||
sql_path: str,
|
||||
empty: bool,
|
||||
) -> None:
|
||||
if not os.path.isfile(sql_path):
|
||||
raise FileNotFoundError(sql_path)
|
||||
|
||||
# Make password available INSIDE the container for psql.
|
||||
docker_env = {"PGPASSWORD": password}
|
||||
|
||||
if empty:
|
||||
drop_sql = r"""
|
||||
DO $$ DECLARE r RECORD;
|
||||
BEGIN
|
||||
FOR r IN (
|
||||
SELECT table_name AS name, 'TABLE' AS type FROM information_schema.tables WHERE table_schema='public'
|
||||
UNION ALL
|
||||
SELECT routine_name AS name, 'FUNCTION' AS type FROM information_schema.routines WHERE specific_schema='public'
|
||||
UNION ALL
|
||||
SELECT sequence_name AS name, 'SEQUENCE' AS type FROM information_schema.sequences WHERE sequence_schema='public'
|
||||
) LOOP
|
||||
EXECUTE format('DROP %s public.%I CASCADE', r.type, r.name);
|
||||
END LOOP;
|
||||
END $$;
|
||||
"""
|
||||
docker_exec(
|
||||
container,
|
||||
["psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", db_name],
|
||||
stdin=drop_sql.encode(),
|
||||
docker_env=docker_env,
|
||||
)
|
||||
|
||||
with open(sql_path, "rb") as f:
|
||||
docker_exec(
|
||||
container,
|
||||
["psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", db_name],
|
||||
stdin=f,
|
||||
docker_env=docker_env,
|
||||
)
|
||||
|
||||
print(f"PostgreSQL restore complete for db '{db_name}'.")
|
||||
37
src/baudolo/restore/files.py
Normal file
37
src/baudolo/restore/files.py
Normal file
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from .run import run, docker_volume_exists
|
||||
|
||||
|
||||
def restore_volume_files(volume_name: str, backup_files_dir: str, *, rsync_image: str) -> int:
|
||||
if not os.path.isdir(backup_files_dir):
|
||||
print(f"ERROR: backup files dir not found: {backup_files_dir}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
if not docker_volume_exists(volume_name):
|
||||
print(f"Volume {volume_name} does not exist. Creating...")
|
||||
run(["docker", "volume", "create", volume_name])
|
||||
else:
|
||||
print(f"Volume {volume_name} already exists.")
|
||||
|
||||
# Keep behavior close to the old script: rsync -avv --delete
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"-v",
|
||||
f"{volume_name}:/recover/",
|
||||
"-v",
|
||||
f"{backup_files_dir}:/backup/",
|
||||
rsync_image,
|
||||
"sh",
|
||||
"-lc",
|
||||
"rsync -avv --delete /backup/ /recover/",
|
||||
]
|
||||
)
|
||||
print("File restore complete.")
|
||||
return 0
|
||||
29
src/baudolo/restore/paths.py
Normal file
29
src/baudolo/restore/paths.py
Normal file
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackupPaths:
|
||||
volume_name: str
|
||||
backup_hash: str
|
||||
version: str
|
||||
repo_name: str
|
||||
backups_dir: str = "/Backups"
|
||||
|
||||
def root(self) -> str:
|
||||
# Always build an absolute path under backups_dir
|
||||
return os.path.join(
|
||||
self.backups_dir,
|
||||
self.backup_hash,
|
||||
self.repo_name,
|
||||
self.version,
|
||||
self.volume_name,
|
||||
)
|
||||
|
||||
def files_dir(self) -> str:
|
||||
return os.path.join(self.root(), "files")
|
||||
|
||||
def sql_file(self, db_name: str) -> str:
|
||||
return os.path.join(self.root(), "sql", f"{db_name}.backup.sql")
|
||||
89
src/baudolo/restore/run.py
Normal file
89
src/baudolo/restore/run.py
Normal file
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def run(
|
||||
cmd: list[str],
|
||||
*,
|
||||
stdin=None,
|
||||
capture: bool = False,
|
||||
env: Optional[dict] = None,
|
||||
) -> subprocess.CompletedProcess:
|
||||
try:
|
||||
kwargs: dict = {
|
||||
"check": True,
|
||||
"capture_output": capture,
|
||||
"env": env,
|
||||
}
|
||||
|
||||
# If stdin is raw data (bytes/str), pass it via input=.
|
||||
# IMPORTANT: when using input=..., do NOT pass stdin=... as well.
|
||||
if isinstance(stdin, (bytes, str)):
|
||||
kwargs["input"] = stdin
|
||||
else:
|
||||
kwargs["stdin"] = stdin
|
||||
|
||||
return subprocess.run(cmd, **kwargs)
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
msg = f"ERROR: command failed ({e.returncode}): {' '.join(cmd)}"
|
||||
print(msg, file=sys.stderr)
|
||||
if e.stdout:
|
||||
try:
|
||||
print(e.stdout.decode(), file=sys.stderr)
|
||||
except Exception:
|
||||
print(e.stdout, file=sys.stderr)
|
||||
if e.stderr:
|
||||
try:
|
||||
print(e.stderr.decode(), file=sys.stderr)
|
||||
except Exception:
|
||||
print(e.stderr, file=sys.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def docker_exec(
|
||||
container: str,
|
||||
argv: list[str],
|
||||
*,
|
||||
stdin=None,
|
||||
capture: bool = False,
|
||||
env: Optional[dict] = None,
|
||||
docker_env: Optional[dict[str, str]] = None,
|
||||
) -> subprocess.CompletedProcess:
|
||||
cmd: list[str] = ["docker", "exec", "-i"]
|
||||
if docker_env:
|
||||
for k, v in docker_env.items():
|
||||
cmd.extend(["-e", f"{k}={v}"])
|
||||
cmd.extend([container, *argv])
|
||||
return run(cmd, stdin=stdin, capture=capture, env=env)
|
||||
|
||||
|
||||
def docker_exec_sh(
|
||||
container: str,
|
||||
script: str,
|
||||
*,
|
||||
stdin=None,
|
||||
capture: bool = False,
|
||||
env: Optional[dict] = None,
|
||||
docker_env: Optional[dict[str, str]] = None,
|
||||
) -> subprocess.CompletedProcess:
|
||||
return docker_exec(
|
||||
container,
|
||||
["sh", "-lc", script],
|
||||
stdin=stdin,
|
||||
capture=capture,
|
||||
env=env,
|
||||
docker_env=docker_env,
|
||||
)
|
||||
|
||||
|
||||
def docker_volume_exists(volume: str) -> bool:
|
||||
p = subprocess.run(
|
||||
["docker", "volume", "inspect", volume],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
return p.returncode == 0
|
||||
0
tests/e2e/__init__.py
Normal file
0
tests/e2e/__init__.py
Normal file
222
tests/e2e/helpers.py
Normal file
222
tests/e2e/helpers.py
Normal file
@@ -0,0 +1,222 @@
|
||||
# tests/e2e/helpers.py
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def run(
|
||||
cmd: list[str],
|
||||
*,
|
||||
capture: bool = True,
|
||||
check: bool = True,
|
||||
cwd: str | None = None,
|
||||
) -> subprocess.CompletedProcess:
|
||||
try:
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
check=check,
|
||||
cwd=cwd,
|
||||
text=True,
|
||||
capture_output=capture,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
# Print captured output so failing E2E tests are "live" / debuggable in CI logs
|
||||
print(">>> command failed:", " ".join(cmd))
|
||||
print(">>> exit code:", e.returncode)
|
||||
if e.stdout:
|
||||
print(">>> STDOUT:\n" + e.stdout)
|
||||
if e.stderr:
|
||||
print(">>> STDERR:\n" + e.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def sh(cmd: str, *, capture: bool = True, check: bool = True) -> subprocess.CompletedProcess:
|
||||
return run(["sh", "-lc", cmd], capture=capture, check=check)
|
||||
|
||||
|
||||
def unique(prefix: str) -> str:
|
||||
return f"{prefix}-{uuid.uuid4().hex[:10]}"
|
||||
|
||||
|
||||
def require_docker() -> None:
|
||||
run(["docker", "version"], capture=True, check=True)
|
||||
|
||||
|
||||
def machine_hash() -> str:
|
||||
out = sh("sha256sum /etc/machine-id | awk '{print $1}'").stdout.strip()
|
||||
if len(out) < 16:
|
||||
raise RuntimeError("Could not determine machine hash from /etc/machine-id")
|
||||
return out
|
||||
|
||||
|
||||
def wait_for_log(container: str, pattern: str, timeout_s: int = 60) -> None:
|
||||
deadline = time.time() + timeout_s
|
||||
while time.time() < deadline:
|
||||
p = run(["docker", "logs", container], capture=True, check=False)
|
||||
if pattern in (p.stdout or ""):
|
||||
return
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"Timed out waiting for log pattern '{pattern}' in {container}")
|
||||
|
||||
|
||||
def wait_for_postgres(container: str, *, user: str = "postgres", timeout_s: int = 90) -> None:
|
||||
"""
|
||||
Docker-outside-of-Docker friendly readiness: check from inside the DB container.
|
||||
"""
|
||||
deadline = time.time() + timeout_s
|
||||
while time.time() < deadline:
|
||||
p = run(
|
||||
["docker", "exec", container, "sh", "-lc", f"pg_isready -U {user} -h localhost"],
|
||||
capture=True,
|
||||
check=False,
|
||||
)
|
||||
if p.returncode == 0:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"Timed out waiting for Postgres readiness in container {container}")
|
||||
|
||||
|
||||
def wait_for_mariadb(container: str, *, root_password: str, timeout_s: int = 90) -> None:
|
||||
"""
|
||||
Liveness probe for MariaDB.
|
||||
|
||||
IMPORTANT (MariaDB 11):
|
||||
Root TCP auth is often restricted (unix_socket auth), so a TCP ping like
|
||||
`mariadb-admin -uroot -p... -h localhost ping` can fail even though the server is up.
|
||||
We therefore check readiness via a socket-based query.
|
||||
"""
|
||||
deadline = time.time() + timeout_s
|
||||
while time.time() < deadline:
|
||||
p = run(
|
||||
["docker", "exec", container, "sh", "-lc", "mariadb -uroot --protocol=socket -e \"SELECT 1;\""],
|
||||
capture=True,
|
||||
check=False,
|
||||
)
|
||||
if p.returncode == 0:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"Timed out waiting for MariaDB readiness in container {container}")
|
||||
|
||||
|
||||
def wait_for_mariadb_sql(container: str, *, user: str, password: str, timeout_s: int = 90) -> None:
|
||||
"""
|
||||
SQL login readiness for the *dedicated test user* over TCP.
|
||||
|
||||
This is separate from wait_for_mariadb(root) because root may be socket-only,
|
||||
while the tests use a normal user that should work via TCP.
|
||||
"""
|
||||
deadline = time.time() + timeout_s
|
||||
while time.time() < deadline:
|
||||
p = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{user} -p{password} -e \"SELECT 1;\"",
|
||||
],
|
||||
capture=True,
|
||||
check=False,
|
||||
)
|
||||
if p.returncode == 0:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"Timed out waiting for MariaDB SQL login readiness in container {container}")
|
||||
|
||||
|
||||
def backup_run(
|
||||
*,
|
||||
backups_dir: str,
|
||||
repo_name: str,
|
||||
compose_dir: str,
|
||||
databases_csv: str,
|
||||
database_containers: list[str],
|
||||
images_no_stop_required: list[str],
|
||||
images_no_backup_required: list[str] | None = None,
|
||||
dump_only: bool = False,
|
||||
) -> None:
|
||||
cmd = [
|
||||
"baudolo",
|
||||
"--compose-dir", compose_dir,
|
||||
"--docker-compose-hard-restart-required", "mailu",
|
||||
"--repo-name", repo_name,
|
||||
"--databases-csv", databases_csv,
|
||||
"--backups-dir", backups_dir,
|
||||
"--database-containers", *database_containers,
|
||||
"--images-no-stop-required", *images_no_stop_required,
|
||||
]
|
||||
if images_no_backup_required:
|
||||
cmd += ["--images-no-backup-required", *images_no_backup_required]
|
||||
if dump_only:
|
||||
cmd += ["--dump-only"]
|
||||
|
||||
try:
|
||||
run(cmd, capture=True, check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(">>> baudolo failed (exit code:", e.returncode, ")")
|
||||
if e.stdout:
|
||||
print(">>> baudolo STDOUT:\n" + e.stdout)
|
||||
if e.stderr:
|
||||
print(">>> baudolo STDERR:\n" + e.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def latest_version_dir(backups_dir: str, repo_name: str) -> tuple[str, str]:
|
||||
"""
|
||||
Returns (hash, version) for the latest backup.
|
||||
"""
|
||||
h = machine_hash()
|
||||
root = Path(backups_dir) / h / repo_name
|
||||
if not root.is_dir():
|
||||
raise FileNotFoundError(str(root))
|
||||
|
||||
versions = sorted([p.name for p in root.iterdir() if p.is_dir()])
|
||||
if not versions:
|
||||
raise RuntimeError(f"No versions found under {root}")
|
||||
return h, versions[-1]
|
||||
|
||||
|
||||
def backup_path(backups_dir: str, repo_name: str, version: str, volume: str) -> Path:
|
||||
h = machine_hash()
|
||||
return Path(backups_dir) / h / repo_name / version / volume
|
||||
|
||||
|
||||
def create_minimal_compose_dir(base: str) -> str:
|
||||
"""
|
||||
baudolo requires --compose-dir. Create an empty dir with one non-compose subdir.
|
||||
"""
|
||||
p = Path(base) / "compose-root"
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
(p / "noop").mkdir(parents=True, exist_ok=True)
|
||||
return str(p)
|
||||
|
||||
|
||||
def write_databases_csv(path: str, rows: list[tuple[str, str, str, str]]) -> None:
|
||||
"""
|
||||
rows: (instance, database, username, password)
|
||||
database may be '' (empty) to trigger pg_dumpall behavior if you want, but here we use db name.
|
||||
"""
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write("instance;database;username;password\n")
|
||||
for inst, db, user, pw in rows:
|
||||
f.write(f"{inst};{db};{user};{pw}\n")
|
||||
|
||||
|
||||
def cleanup_docker(*, containers: list[str], volumes: list[str]) -> None:
|
||||
for c in containers:
|
||||
run(["docker", "rm", "-f", c], capture=True, check=False)
|
||||
for v in volumes:
|
||||
run(["docker", "volume", "rm", "-f", v], capture=True, check=False)
|
||||
|
||||
|
||||
def ensure_empty_dir(path: str) -> None:
|
||||
p = Path(path)
|
||||
if p.exists():
|
||||
shutil.rmtree(p)
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
94
tests/e2e/test_e2e_files_full.py
Normal file
94
tests/e2e/test_e2e_files_full.py
Normal file
@@ -0,0 +1,94 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
)
|
||||
|
||||
|
||||
class TestE2EFilesFull(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-files-full")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.volume_src = f"{cls.prefix}-vol-src"
|
||||
cls.volume_dst = f"{cls.prefix}-vol-dst"
|
||||
cls.containers = []
|
||||
cls.volumes = [cls.volume_src, cls.volume_dst]
|
||||
|
||||
# create source volume with a file
|
||||
run(["docker", "volume", "create", cls.volume_src])
|
||||
run([
|
||||
"docker", "run", "--rm",
|
||||
"-v", f"{cls.volume_src}:/data",
|
||||
"alpine:3.20",
|
||||
"sh", "-lc", "mkdir -p /data && echo 'hello' > /data/hello.txt",
|
||||
])
|
||||
|
||||
# databases.csv (unused, but required by CLI)
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [])
|
||||
|
||||
# Run backup (files should be copied)
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=["dummy-db"],
|
||||
images_no_stop_required=["alpine", "postgres", "mariadb", "mysql"],
|
||||
)
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def test_files_backup_exists(self) -> None:
|
||||
p = (
|
||||
backup_path(
|
||||
self.backups_dir,
|
||||
self.repo_name,
|
||||
self.version,
|
||||
self.volume_src,
|
||||
)
|
||||
/ "files"
|
||||
/ "hello.txt"
|
||||
)
|
||||
self.assertTrue(p.is_file(), f"Expected backed up file at: {p}")
|
||||
|
||||
def test_restore_files_into_new_volume(self) -> None:
|
||||
# restore files from volume_src backup into volume_dst
|
||||
run([
|
||||
"baudolo-restore", "files",
|
||||
self.volume_dst, self.hash, self.version,
|
||||
"--backups-dir", self.backups_dir,
|
||||
"--repo-name", self.repo_name,
|
||||
"--source-volume", self.volume_src,
|
||||
"--rsync-image", "ghcr.io/kevinveenbirkenbach/alpine-rsync",
|
||||
])
|
||||
|
||||
# verify restored file exists in dst volume
|
||||
p = run([
|
||||
"docker", "run", "--rm",
|
||||
"-v", f"{self.volume_dst}:/data",
|
||||
"alpine:3.20",
|
||||
"sh", "-lc", "cat /data/hello.txt",
|
||||
])
|
||||
self.assertEqual((p.stdout or "").strip(), "hello")
|
||||
72
tests/e2e/test_e2e_files_no_copy.py
Normal file
72
tests/e2e/test_e2e_files_no_copy.py
Normal file
@@ -0,0 +1,72 @@
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
)
|
||||
|
||||
|
||||
class TestE2EFilesNoCopy(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-files-nocopy")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.volume_src = f"{cls.prefix}-vol-src"
|
||||
cls.volume_dst = f"{cls.prefix}-vol-dst"
|
||||
cls.containers = []
|
||||
cls.volumes = [cls.volume_src, cls.volume_dst]
|
||||
|
||||
run(["docker", "volume", "create", cls.volume_src])
|
||||
run([
|
||||
"docker", "run", "--rm",
|
||||
"-v", f"{cls.volume_src}:/data",
|
||||
"alpine:3.20",
|
||||
"sh", "-lc", "echo 'hello' > /data/hello.txt",
|
||||
])
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [])
|
||||
|
||||
# dump-only => NO file rsync backups
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=["dummy-db"],
|
||||
images_no_stop_required=["alpine", "postgres", "mariadb", "mysql"],
|
||||
dump_only=True,
|
||||
)
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def test_files_backup_not_present(self) -> None:
|
||||
p = backup_path(self.backups_dir, self.repo_name, self.version, self.volume_src) / "files"
|
||||
self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
|
||||
|
||||
def test_restore_files_fails_expected(self) -> None:
|
||||
p = run([
|
||||
"baudolo-restore", "files",
|
||||
self.volume_dst, self.hash, self.version,
|
||||
"--backups-dir", self.backups_dir,
|
||||
"--repo-name", self.repo_name,
|
||||
], check=False)
|
||||
self.assertEqual(p.returncode, 2, f"Expected exitcode 2, got {p.returncode}\nSTDOUT={p.stdout}\nSTDERR={p.stderr}")
|
||||
155
tests/e2e/test_e2e_mariadb_full.py
Normal file
155
tests/e2e/test_e2e_mariadb_full.py
Normal file
@@ -0,0 +1,155 @@
|
||||
# tests/e2e/test_e2e_mariadb_full.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
wait_for_mariadb,
|
||||
wait_for_mariadb_sql,
|
||||
)
|
||||
|
||||
|
||||
class TestE2EMariaDBFull(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-mariadb-full")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.db_container = f"{cls.prefix}-mariadb"
|
||||
cls.db_volume = f"{cls.prefix}-mariadb-vol"
|
||||
cls.containers = [cls.db_container]
|
||||
cls.volumes = [cls.db_volume]
|
||||
|
||||
cls.db_name = "appdb"
|
||||
cls.db_user = "test"
|
||||
cls.db_password = "testpw"
|
||||
cls.root_password = "rootpw"
|
||||
|
||||
run(["docker", "volume", "create", cls.db_volume])
|
||||
|
||||
# Start MariaDB with a dedicated TCP-capable user for tests.
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.db_container,
|
||||
"-e",
|
||||
f"MARIADB_ROOT_PASSWORD={cls.root_password}",
|
||||
"-e",
|
||||
f"MARIADB_DATABASE={cls.db_name}",
|
||||
"-e",
|
||||
f"MARIADB_USER={cls.db_user}",
|
||||
"-e",
|
||||
f"MARIADB_PASSWORD={cls.db_password}",
|
||||
"-v",
|
||||
f"{cls.db_volume}:/var/lib/mysql",
|
||||
"mariadb:11",
|
||||
]
|
||||
)
|
||||
|
||||
# Liveness + actual SQL login readiness (TCP)
|
||||
wait_for_mariadb(cls.db_container, root_password=cls.root_password, timeout_s=90)
|
||||
wait_for_mariadb_sql(cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90)
|
||||
|
||||
# Create table + data via the dedicated user (TCP)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f"-e \"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); "
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
||||
]
|
||||
)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
# IMPORTANT: baudolo backup expects credentials for the DB dump.
|
||||
write_databases_csv(cls.databases_csv, [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)])
|
||||
|
||||
# Backup with file+dump
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.db_container],
|
||||
images_no_stop_required=["mariadb", "mysql", "alpine", "postgres"],
|
||||
)
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
# Wipe DB via the dedicated user (TCP)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f"-e \"DROP TABLE {cls.db_name}.t;\"",
|
||||
]
|
||||
)
|
||||
|
||||
# Restore DB (uses baudolo-restore which execs mysql/mariadb inside the container)
|
||||
run(
|
||||
[
|
||||
"baudolo-restore",
|
||||
"mariadb",
|
||||
cls.db_volume,
|
||||
cls.hash,
|
||||
cls.version,
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--repo-name",
|
||||
cls.repo_name,
|
||||
"--container",
|
||||
cls.db_container,
|
||||
"--db-name",
|
||||
cls.db_name,
|
||||
"--db-user",
|
||||
cls.db_user,
|
||||
"--db-password",
|
||||
cls.db_password,
|
||||
"--empty",
|
||||
]
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def test_dump_file_exists(self) -> None:
|
||||
p = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume) / "sql" / f"{self.db_name}.backup.sql"
|
||||
self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
|
||||
|
||||
def test_data_restored(self) -> None:
|
||||
p = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
self.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f"-N -e \"SELECT v FROM {self.db_name}.t WHERE id=1;\"",
|
||||
]
|
||||
)
|
||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||
153
tests/e2e/test_e2e_mariadb_no_copy.py
Normal file
153
tests/e2e/test_e2e_mariadb_no_copy.py
Normal file
@@ -0,0 +1,153 @@
|
||||
# tests/e2e/test_e2e_mariadb_no_copy.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
wait_for_mariadb,
|
||||
wait_for_mariadb_sql,
|
||||
)
|
||||
|
||||
|
||||
class TestE2EMariaDBNoCopy(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-mariadb-nocopy")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.db_container = f"{cls.prefix}-mariadb"
|
||||
cls.db_volume = f"{cls.prefix}-mariadb-vol"
|
||||
cls.containers = [cls.db_container]
|
||||
cls.volumes = [cls.db_volume]
|
||||
|
||||
cls.db_name = "appdb"
|
||||
cls.db_user = "test"
|
||||
cls.db_password = "testpw"
|
||||
cls.root_password = "rootpw"
|
||||
|
||||
run(["docker", "volume", "create", cls.db_volume])
|
||||
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
cls.db_container,
|
||||
"-e",
|
||||
f"MARIADB_ROOT_PASSWORD={cls.root_password}",
|
||||
"-e",
|
||||
f"MARIADB_DATABASE={cls.db_name}",
|
||||
"-e",
|
||||
f"MARIADB_USER={cls.db_user}",
|
||||
"-e",
|
||||
f"MARIADB_PASSWORD={cls.db_password}",
|
||||
"-v",
|
||||
f"{cls.db_volume}:/var/lib/mysql",
|
||||
"mariadb:11",
|
||||
]
|
||||
)
|
||||
|
||||
wait_for_mariadb(cls.db_container, root_password=cls.root_password, timeout_s=90)
|
||||
wait_for_mariadb_sql(cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90)
|
||||
|
||||
# Create table + data (TCP)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f"-e \"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); "
|
||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
||||
]
|
||||
)
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)])
|
||||
|
||||
# dump-only => no files
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.db_container],
|
||||
images_no_stop_required=["mariadb", "mysql", "alpine", "postgres"],
|
||||
dump_only=True,
|
||||
)
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
# Wipe table (TCP)
|
||||
run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||
f"-e \"DROP TABLE {cls.db_name}.t;\"",
|
||||
]
|
||||
)
|
||||
|
||||
# Restore DB
|
||||
run(
|
||||
[
|
||||
"baudolo-restore",
|
||||
"mariadb",
|
||||
cls.db_volume,
|
||||
cls.hash,
|
||||
cls.version,
|
||||
"--backups-dir",
|
||||
cls.backups_dir,
|
||||
"--repo-name",
|
||||
cls.repo_name,
|
||||
"--container",
|
||||
cls.db_container,
|
||||
"--db-name",
|
||||
cls.db_name,
|
||||
"--db-user",
|
||||
cls.db_user,
|
||||
"--db-password",
|
||||
cls.db_password,
|
||||
"--empty",
|
||||
]
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def test_files_backup_not_present(self) -> None:
|
||||
p = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume) / "files"
|
||||
self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
|
||||
|
||||
def test_data_restored(self) -> None:
|
||||
p = run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
self.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||
f"-N -e \"SELECT v FROM {self.db_name}.t WHERE id=1;\"",
|
||||
]
|
||||
)
|
||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||
102
tests/e2e/test_e2e_postgres_full.py
Normal file
102
tests/e2e/test_e2e_postgres_full.py
Normal file
@@ -0,0 +1,102 @@
|
||||
# tests/e2e/test_e2e_postgres_full.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
wait_for_postgres,
|
||||
)
|
||||
|
||||
|
||||
class TestE2EPostgresFull(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-postgres-full")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.pg_container = f"{cls.prefix}-pg"
|
||||
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||
cls.containers = [cls.pg_container]
|
||||
cls.volumes = [cls.pg_volume]
|
||||
|
||||
run(["docker", "volume", "create", cls.pg_volume])
|
||||
|
||||
run([
|
||||
"docker", "run", "-d",
|
||||
"--name", cls.pg_container,
|
||||
"-e", "POSTGRES_PASSWORD=pgpw",
|
||||
"-e", "POSTGRES_DB=appdb",
|
||||
"-e", "POSTGRES_USER=postgres",
|
||||
"-v", f"{cls.pg_volume}:/var/lib/postgresql/data",
|
||||
"postgres:16",
|
||||
])
|
||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||
|
||||
# Create a table + data
|
||||
run([
|
||||
"docker", "exec", cls.pg_container,
|
||||
"sh", "-lc",
|
||||
"psql -U postgres -d appdb -c \"CREATE TABLE t (id int primary key, v text); INSERT INTO t VALUES (1,'ok');\"",
|
||||
])
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [(cls.pg_container, "appdb", "postgres", "pgpw")])
|
||||
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.pg_container],
|
||||
images_no_stop_required=["postgres", "mariadb", "mysql", "alpine"],
|
||||
)
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
# Wipe schema
|
||||
run([
|
||||
"docker", "exec", cls.pg_container,
|
||||
"sh", "-lc",
|
||||
"psql -U postgres -d appdb -c \"DROP TABLE t;\"",
|
||||
])
|
||||
|
||||
# Restore
|
||||
run([
|
||||
"baudolo-restore", "postgres",
|
||||
cls.pg_volume, cls.hash, cls.version,
|
||||
"--backups-dir", cls.backups_dir,
|
||||
"--repo-name", cls.repo_name,
|
||||
"--container", cls.pg_container,
|
||||
"--db-name", "appdb",
|
||||
"--db-user", "postgres",
|
||||
"--db-password", "pgpw",
|
||||
"--empty",
|
||||
])
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def test_dump_file_exists(self) -> None:
|
||||
p = backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume) / "sql" / "appdb.backup.sql"
|
||||
self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
|
||||
|
||||
def test_data_restored(self) -> None:
|
||||
p = run([
|
||||
"docker", "exec", self.pg_container,
|
||||
"sh", "-lc",
|
||||
"psql -U postgres -d appdb -t -c \"SELECT v FROM t WHERE id=1;\"",
|
||||
])
|
||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||
99
tests/e2e/test_e2e_postgres_no_copy.py
Normal file
99
tests/e2e/test_e2e_postgres_no_copy.py
Normal file
@@ -0,0 +1,99 @@
|
||||
# tests/e2e/test_e2e_postgres_no_copy.py
|
||||
import unittest
|
||||
|
||||
from .helpers import (
|
||||
backup_run,
|
||||
backup_path,
|
||||
cleanup_docker,
|
||||
create_minimal_compose_dir,
|
||||
ensure_empty_dir,
|
||||
latest_version_dir,
|
||||
require_docker,
|
||||
unique,
|
||||
write_databases_csv,
|
||||
run,
|
||||
wait_for_postgres,
|
||||
)
|
||||
|
||||
|
||||
class TestE2EPostgresNoCopy(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
require_docker()
|
||||
cls.prefix = unique("baudolo-e2e-postgres-nocopy")
|
||||
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||
ensure_empty_dir(cls.backups_dir)
|
||||
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||
cls.repo_name = cls.prefix
|
||||
|
||||
cls.pg_container = f"{cls.prefix}-pg"
|
||||
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||
cls.containers = [cls.pg_container]
|
||||
cls.volumes = [cls.pg_volume]
|
||||
|
||||
run(["docker", "volume", "create", cls.pg_volume])
|
||||
run([
|
||||
"docker", "run", "-d",
|
||||
"--name", cls.pg_container,
|
||||
"-e", "POSTGRES_PASSWORD=pgpw",
|
||||
"-e", "POSTGRES_DB=appdb",
|
||||
"-e", "POSTGRES_USER=postgres",
|
||||
"-v", f"{cls.pg_volume}:/var/lib/postgresql/data",
|
||||
"postgres:16",
|
||||
])
|
||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||
|
||||
run([
|
||||
"docker", "exec", cls.pg_container,
|
||||
"sh", "-lc",
|
||||
"psql -U postgres -d appdb -c \"CREATE TABLE t (id int primary key, v text); INSERT INTO t VALUES (1,'ok');\"",
|
||||
])
|
||||
|
||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||
write_databases_csv(cls.databases_csv, [(cls.pg_container, "appdb", "postgres", "pgpw")])
|
||||
|
||||
backup_run(
|
||||
backups_dir=cls.backups_dir,
|
||||
repo_name=cls.repo_name,
|
||||
compose_dir=cls.compose_dir,
|
||||
databases_csv=cls.databases_csv,
|
||||
database_containers=[cls.pg_container],
|
||||
images_no_stop_required=["postgres", "mariadb", "mysql", "alpine"],
|
||||
dump_only=True,
|
||||
)
|
||||
|
||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||
|
||||
run([
|
||||
"docker", "exec", cls.pg_container,
|
||||
"sh", "-lc",
|
||||
"psql -U postgres -d appdb -c \"DROP TABLE t;\"",
|
||||
])
|
||||
|
||||
run([
|
||||
"baudolo-restore", "postgres",
|
||||
cls.pg_volume, cls.hash, cls.version,
|
||||
"--backups-dir", cls.backups_dir,
|
||||
"--repo-name", cls.repo_name,
|
||||
"--container", cls.pg_container,
|
||||
"--db-name", "appdb",
|
||||
"--db-user", "postgres",
|
||||
"--db-password", "pgpw",
|
||||
"--empty",
|
||||
])
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||
|
||||
def test_files_backup_not_present(self) -> None:
|
||||
p = backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume) / "files"
|
||||
self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
|
||||
|
||||
def test_data_restored(self) -> None:
|
||||
p = run([
|
||||
"docker", "exec", self.pg_container,
|
||||
"sh", "-lc",
|
||||
"psql -U postgres -d appdb -t -c \"SELECT v FROM t WHERE id=1;\"",
|
||||
])
|
||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||
0
tests/integration/__init__.py
Normal file
0
tests/integration/__init__.py
Normal file
88
tests/integration/test_seed_integration.py
Normal file
88
tests/integration/test_seed_integration.py
Normal file
@@ -0,0 +1,88 @@
|
||||
import csv
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def run_seed(csv_path: Path, instance: str, database: str, username: str, password: str = "") -> subprocess.CompletedProcess:
|
||||
# Run the real CLI module (integration-style).
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"baudolo.seed",
|
||||
str(csv_path),
|
||||
instance,
|
||||
database,
|
||||
username,
|
||||
password,
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def read_csv_semicolon(path: Path) -> list[dict]:
|
||||
with path.open("r", encoding="utf-8", newline="") as f:
|
||||
reader = csv.DictReader(f, delimiter=";")
|
||||
return list(reader)
|
||||
|
||||
|
||||
class TestSeedIntegration(unittest.TestCase):
|
||||
def test_creates_file_and_adds_entry_when_missing(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td) / "databases.csv"
|
||||
self.assertFalse(p.exists())
|
||||
|
||||
cp = run_seed(p, "docker.test", "appdb", "alice", "secret")
|
||||
|
||||
self.assertEqual(cp.returncode, 0, cp.stderr)
|
||||
self.assertTrue(p.exists())
|
||||
|
||||
rows = read_csv_semicolon(p)
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["instance"], "docker.test")
|
||||
self.assertEqual(rows[0]["database"], "appdb")
|
||||
self.assertEqual(rows[0]["username"], "alice")
|
||||
self.assertEqual(rows[0]["password"], "secret")
|
||||
|
||||
def test_replaces_existing_entry_same_keys(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td) / "databases.csv"
|
||||
|
||||
# First add
|
||||
run_seed(p, "docker.test", "appdb", "alice", "oldpw")
|
||||
rows = read_csv_semicolon(p)
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["password"], "oldpw")
|
||||
|
||||
# Replace (same instance+database+username)
|
||||
run_seed(p, "docker.test", "appdb", "alice", "newpw")
|
||||
rows = read_csv_semicolon(p)
|
||||
|
||||
self.assertEqual(len(rows), 1, "Expected replacement, not a duplicate row")
|
||||
self.assertEqual(rows[0]["instance"], "docker.test")
|
||||
self.assertEqual(rows[0]["database"], "appdb")
|
||||
self.assertEqual(rows[0]["username"], "alice")
|
||||
self.assertEqual(rows[0]["password"], "newpw")
|
||||
|
||||
def test_database_empty_string_matches_existing_empty_database(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td) / "databases.csv"
|
||||
|
||||
# Add with empty database
|
||||
run_seed(p, "docker.test", "", "alice", "pw1")
|
||||
rows = read_csv_semicolon(p)
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["database"], "")
|
||||
|
||||
# Replace with empty database again
|
||||
run_seed(p, "docker.test", "", "alice", "pw2")
|
||||
rows = read_csv_semicolon(p)
|
||||
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["database"], "")
|
||||
self.assertEqual(rows[0]["password"], "pw2")
|
||||
@@ -1,64 +1,36 @@
|
||||
# tests/unit/test_backup.py
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import importlib.util
|
||||
import sys
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
# Prevent actual directory creation in backup script import
|
||||
dummy_mkdir = lambda self, *args, **kwargs: None
|
||||
original_mkdir = pathlib.Path.mkdir
|
||||
pathlib.Path.mkdir = dummy_mkdir
|
||||
from baudolo.backup.app import requires_stop
|
||||
|
||||
# Create a virtual databases.csv in the project root for the module import
|
||||
test_dir = os.path.dirname(__file__)
|
||||
project_root = os.path.abspath(os.path.join(test_dir, '../../'))
|
||||
sys.path.insert(0, project_root)
|
||||
db_csv_path = os.path.join(project_root, 'databases.csv')
|
||||
with open(db_csv_path, 'w') as f:
|
||||
f.write('instance;database;username;password\n')
|
||||
|
||||
# Dynamically load the hyphenated script as module 'backup'
|
||||
script_path = os.path.join(project_root, 'backup-docker-to-local.py')
|
||||
spec = importlib.util.spec_from_file_location('backup', script_path)
|
||||
backup = importlib.util.module_from_spec(spec)
|
||||
sys.modules['backup'] = backup
|
||||
spec.loader.exec_module(backup)
|
||||
class TestRequiresStop(unittest.TestCase):
|
||||
@patch("baudolo.backup.app.get_image_info")
|
||||
def test_requires_stop_false_when_all_images_are_whitelisted(self, mock_get_image_info):
|
||||
# All containers use images containing allowed substrings
|
||||
mock_get_image_info.side_effect = [
|
||||
"repo/mastodon:v4",
|
||||
"repo/wordpress:latest",
|
||||
]
|
||||
containers = ["c1", "c2"]
|
||||
whitelist = ["mastodon", "wordpress"]
|
||||
self.assertFalse(requires_stop(containers, whitelist))
|
||||
|
||||
# Restore original mkdir
|
||||
pathlib.Path.mkdir = original_mkdir
|
||||
@patch("baudolo.backup.app.get_image_info")
|
||||
def test_requires_stop_true_when_any_image_is_not_whitelisted(self, mock_get_image_info):
|
||||
mock_get_image_info.side_effect = [
|
||||
"repo/mastodon:v4",
|
||||
"repo/nginx:latest",
|
||||
]
|
||||
containers = ["c1", "c2"]
|
||||
whitelist = ["mastodon", "wordpress"]
|
||||
self.assertTrue(requires_stop(containers, whitelist))
|
||||
|
||||
class TestIsImageWhitelisted(unittest.TestCase):
|
||||
@patch('backup.get_image_info')
|
||||
def test_returns_true_when_image_matches(self, mock_get_image_info):
|
||||
# Simulate a container image containing 'mastodon'
|
||||
mock_get_image_info.return_value = ['repo/mastodon:v4']
|
||||
images = ['mastodon', 'wordpress']
|
||||
self.assertTrue(
|
||||
backup.is_image_whitelisted('any_container', images),
|
||||
"Should return True when at least one image substring matches"
|
||||
)
|
||||
@patch("baudolo.backup.app.get_image_info")
|
||||
def test_requires_stop_true_when_whitelist_empty(self, mock_get_image_info):
|
||||
mock_get_image_info.return_value = "repo/anything:latest"
|
||||
self.assertTrue(requires_stop(["c1"], []))
|
||||
|
||||
@patch('backup.get_image_info')
|
||||
def test_returns_false_when_no_image_matches(self, mock_get_image_info):
|
||||
# Simulate a container image without matching substrings
|
||||
mock_get_image_info.return_value = ['repo/nginx:latest']
|
||||
images = ['mastodon', 'wordpress']
|
||||
self.assertFalse(
|
||||
backup.is_image_whitelisted('any_container', images),
|
||||
"Should return False when no image substring matches"
|
||||
)
|
||||
|
||||
@patch('backup.get_image_info')
|
||||
def test_returns_false_with_empty_image_list(self, mock_get_image_info):
|
||||
# Even if get_image_info returns something, an empty list yields False
|
||||
mock_get_image_info.return_value = ['repo/element:1.0']
|
||||
self.assertFalse(
|
||||
backup.is_image_whitelisted('any_container', []),
|
||||
"Should return False when the images list is empty"
|
||||
)
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user