Release version 1.2.0

- read databases.csv with stable types (dtype=str, keep_default_na=False)
- validate database field: require '*' or concrete name (no empty/NaN)
- support Postgres cluster dumps via '*' entries (pg_dumpall)
- write SQL dumps atomically to avoid partial/empty files
- early-skip fully ignored volumes before creating backup directories
- update seed CLI to enforce new contract and update by (instance,database)
- adjust tests: sql dir naming + add E2E coverage for early-skip and '*' seeding

.github/workflows/ci.yml

@@ -0,0 +1,91 @@
+name: CI (make tests, stable, publish)
+
+on:
+  push:
+    branches: ["**"]
+    tags: ["v*.*.*"]   # SemVer tags like v1.2.3
+  pull_request:
+
+permissions:
+  contents: write      # push/update 'stable' tag
+  packages: write      # push to GHCR
+
+env:
+  IMAGE_NAME: baudolo
+  REGISTRY: ghcr.io
+  IMAGE_REPO: ${{ github.repository }}
+
+jobs:
+  test:
+    name: make test
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Show docker info
+        run: |
+          docker version
+          docker info
+
+      - name: Run all tests via Makefile
+        run: |
+          make test
+
+      - name: Upload E2E artifacts (always)
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-artifacts
+          path: artifacts
+          if-no-files-found: ignore
+
+  stable_and_publish:
+    name: Mark stable + publish image (SemVer tags only)
+    needs: [test]
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
+
+    steps:
+      - name: Checkout (full history for tags)
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Derive version from tag
+        id: ver
+        run: |
+          TAG="${GITHUB_REF#refs/tags/}"   # v1.2.3
+          echo "tag=${TAG}" >> "$GITHUB_OUTPUT"
+
+      - name: Mark 'stable' git tag (force update)
+        run: |
+          git config user.name  "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git tag -f stable "${GITHUB_SHA}"
+          git push -f origin stable
+
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build image (Makefile)
+        run: |
+          make build
+
+      - name: Tag image for registry
+        run: |
+          # local image built by Makefile is: baudolo:local
+          docker tag "${IMAGE_NAME}:local" "${REGISTRY}/${IMAGE_REPO}:${{ steps.ver.outputs.tag }}"
+          docker tag "${IMAGE_NAME}:local" "${REGISTRY}/${IMAGE_REPO}:stable"
+          docker tag "${IMAGE_NAME}:local" "${REGISTRY}/${IMAGE_REPO}:sha-${GITHUB_SHA::12}"
+
+      - name: Push image
+        run: |
+          docker push "${REGISTRY}/${IMAGE_REPO}:${{ steps.ver.outputs.tag }}"
+          docker push "${REGISTRY}/${IMAGE_REPO}:stable"
+          docker push "${REGISTRY}/${IMAGE_REPO}:sha-${GITHUB_SHA::12}"

.gitignore

@@ -1,2 +1,4 @@
-databases.csv
+__pycache__
-__pycache__
+artifacts/
+*.egg-info
+dist/

.travis.yml

@@ -1,2 +0,0 @@
-language: shell
-script: shellcheck $(find . -type f -name '*.sh')

CHANGELOG.md

@@ -0,0 +1,26 @@
+## [1.2.0] - 2025-12-29
+
+* * Introduced **`--dump-only-sql`** mode for reliable, SQL-only database backups (replaces `--dump-only`).
+* Database configuration in `databases.csv` is now **strict and explicit** (`*` or concrete database name only).
+* **PostgreSQL cluster backups** are supported via `*`.
+* SQL dumps are written **atomically** to avoid corrupted or empty files.
+* Backups are **smarter and faster**: ignored volumes are skipped early, file backups run only when needed.
+* Improved reliability through expanded end-to-end tests and safer defaults.
+
+
+## [1.1.1] - 2025-12-28
+
+* * **Backup:** In ***--dump-only-sql*** mode, fall back to file backups with a warning when no database dump can be produced (e.g. missing `databases.csv` entry).
+
+
+## [1.1.0] - 2025-12-28
+
+* * **Backup:** Log a warning and skip database dumps when no databases.csv entry is present instead of raising an exception; introduce module-level logging and apply formatting cleanups across backup/restore code and tests.
+* **CLI:** Switch to an FHS-compliant default backup directory (/var/lib/backup) and use a stable default repository name instead of dynamic detection.
+* **Maintenance:** Update mirror configuration and ignore generated .egg-info files.
+
+
+## [1.0.0] - 2025-12-27
+
+* Official Release 🥳
+

Dockerfile

@@ -0,0 +1,34 @@
+# syntax=docker/dockerfile:1
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Runtime + build essentials:
+# - rsync: required for file backup/restore
+# - ca-certificates: TLS
+# - docker-cli: needed if you want to control the host Docker engine (via /var/run/docker.sock mount)
+# - make: to delegate install logic to Makefile
+#
+# Notes:
+# - On Debian slim, the docker client package is typically "docker.io".
+# - If you only want restore-without-docker, you can drop docker.io later.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    make \
+    rsync \
+    ca-certificates \
+    docker-cli \
+  && rm -rf /var/lib/apt/lists/*
+
+# Fail fast if docker client is missing
+RUN command -v docker
+
+COPY . .
+
+# All install decisions are handled by the Makefile.
+RUN make install
+
+# Sensible defaults (can be overridden at runtime)
+ENV PYTHONUNBUFFERED=1
+
+# Default: show CLI help
+CMD ["baudolo", "--help"]

MIRRORS

@@ -0,0 +1,4 @@
+git@github.com:kevinveenbirkenbach/backup-docker-to-local.git
+ssh://git@git.veen.world:2201/kevinveenbirkenbach/backup-docker-to-local.git
+ssh://git@code.infinito.nexus:2201/kevinveenbirkenbach/backup-docker-to-local.git
+https://pypi.org/project/backup-docker-to-local/

Makefile

@@ -1,4 +1,57 @@
-.PHONY: test
+.PHONY: install build \
+		test-e2e test test-unit test-integration
 
-test:
+# Default python if no venv is active
-	python -m unittest discover -s tests/unit -p "test_*.py"
+PY_DEFAULT ?= python3
+
+IMAGE_NAME ?= baudolo
+IMAGE_TAG  ?= local
+IMAGE      := $(IMAGE_NAME):$(IMAGE_TAG)
+
+install:
+	@set -eu; \
+	PY="$(PY_DEFAULT)"; \
+	if [ -n "$${VIRTUAL_ENV:-}" ] && [ -x "$${VIRTUAL_ENV}/bin/python" ]; then \
+		PY="$${VIRTUAL_ENV}/bin/python"; \
+	fi; \
+	echo ">>> Using python: $$PY"; \
+	"$$PY" -m pip install --upgrade pip; \
+	"$$PY" -m pip install -e .; \
+	command -v baudolo >/dev/null 2>&1 || { \
+		echo "ERROR: baudolo not found on PATH after install"; \
+		exit 2; \
+	}; \
+	baudolo --help >/dev/null 2>&1 || true
+
+# ------------------------------------------------------------
+# Build the baudolo Docker image
+# ------------------------------------------------------------
+build:
+	@echo ">> Building Docker image $(IMAGE)"
+	docker build -t $(IMAGE) .
+
+clean:
+	git clean -fdX .
+
+# ------------------------------------------------------------
+# Run E2E tests inside the container (Docker socket required)
+# ------------------------------------------------------------
+# E2E via isolated Docker-in-Docker (DinD)
+# - depends on local image build
+# - starts a DinD daemon container on a dedicated network
+# - loads the freshly built image into DinD
+# - runs the unittest suite inside a container that talks to DinD via DOCKER_HOST
+test-e2e: clean build
+	@bash scripts/test-e2e.sh
+
+test: test-unit test-integration test-e2e
+
+test-unit: clean build
+	@echo ">> Running unit tests"
+	@docker run --rm -t $(IMAGE) \
+	  sh -lc 'python -m unittest discover -t . -s tests/unit -p "test_*.py" -v'
+
+test-integration: clean build
+	@echo ">> Running integration tests"
+	@docker run --rm -t $(IMAGE) \
+	  sh -lc 'python -m unittest discover -t . -s tests/integration -p "test_*.py" -v'

README.md

@@ -1,80 +1,196 @@
-# Backup Docker Volumes to Local (baudolo) 📦🔄
+# baudolo – Deterministic Backup & Restore for Docker Volumes 📦🔄
-[![GitHub Sponsors](https://img.shields.io/badge/Sponsor-GitHub%20Sponsors-blue?logo=github)](https://github.com/sponsors/kevinveenbirkenbach) [![Patreon](https://img.shields.io/badge/Support-Patreon-orange?logo=patreon)](https://www.patreon.com/c/kevinveenbirkenbach) [![Buy Me a Coffee](https://img.shields.io/badge/Buy%20me%20a%20Coffee-Funding-yellow?logo=buymeacoffee)](https://buymeacoffee.com/kevinveenbirkenbach) [![PayPal](https://img.shields.io/badge/Donate-PayPal-blue?logo=paypal)](https://s.veen.world/paypaldonate)
+[![GitHub Sponsors](https://img.shields.io/badge/Sponsor-GitHub%20Sponsors-blue?logo=github)](https://github.com/sponsors/kevinveenbirkenbach) [![Patreon](https://img.shields.io/badge/Support-Patreon-orange?logo=patreon)](https://www.patreon.com/c/kevinveenbirkenbach) [![Buy Me a Coffee](https://img.shields.io/badge/Buy%20me%20a%20Coffee-Funding-yellow?logo=buymeacoffee)](https://buymeacoffee.com/kevinveenbirkenbach) [![PayPal](https://img.shields.io/badge/Donate-PayPal-blue?logo=paypal)](https://s.veen.world/paypaldonate) [![License: AGPL v3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0) [![Docker Version](https://img.shields.io/badge/Docker-Yes-blue.svg)](https://www.docker.com) [![Python Version](https://img.shields.io/badge/Python-3.x-blue.svg)](https://www.python.org) [![GitHub stars](https://img.shields.io/github/stars/kevinveenbirkenbach/backup-docker-to-local.svg?style=social)](https://github.com/kevinveenbirkenbach/backup-docker-to-local/stargazers)
 
 
-**Backup Docker Volumes to Local** is a set of Python and shell scripts that enable you to perform incremental backups of all your Docker volumes using rsync. It is designed to integrate seamlessly with [Kevin's Package Manager](https://github.com/kevinveenbirkenbach/package-manager) under the alias **baudolo**, making it easy to install and manage. The tool supports both file and database recoveries with a clear, automated backup scheme.
+`baudolo` is a backup and restore system for Docker volumes with
+**mandatory file backups** and **explicit, deterministic database dumps**.
+It is designed for environments with many Docker services where:
+- file-level backups must always exist
+- database dumps must be intentional, predictable, and auditable
 
-[![License: AGPL v3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0) [![Docker Version](https://img.shields.io/badge/Docker-Yes-blue.svg)](https://www.docker.com) [![Python Version](https://img.shields.io/badge/Python-3.x-blue.svg)](https://www.python.org) [![GitHub stars](https://img.shields.io/github/stars/kevinveenbirkenbach/backup-docker-to-local.svg?style=social)](https://github.com/kevinveenbirkenbach/backup-docker-to-local/stargazers)
+## ✨ Key Features
 
-## 🎯 Goal
+- 📦 Incremental Docker volume backups using `rsync --link-dest`
+- 🗄 Optional SQL dumps for:
+  - PostgreSQL
+  - MariaDB / MySQL
+- 🌱 Explicit database definition for SQL backups (no auto-discovery)
+- 🧾 Backup integrity stamping via `dirval` (Python API)
+- ⏸ Automatic container stop/start when required for consistency
+- 🚫 Whitelisting of containers that do not require stopping
+- ♻️ Modular, maintainable Python architecture
 
-This project automates the backup of Docker volumes using incremental backups (rsync) and supports recovering both files and database dumps (MariaDB/PostgreSQL). A robust directory stamping mechanism ensures data integrity, and the tool also handles restarting Docker Compose services when necessary.
 
-## 🚀 Features
+## 🧠 Core Concept (Important!)
 
-- **Incremental Backups:** Uses rsync with `--link-dest` for efficient, versioned backups.
+`baudolo` **separates file backups from database dumps**.
-- **Database Backup Support:** Backs up MariaDB and PostgreSQL databases from running containers.
-- **Volume Recovery:** Provides scripts to recover volumes and databases from backups.
-- **Docker Compose Integration:** Option to automatically restart Docker Compose services after backup.
-- **Flexible Configuration:** Easily integrated with your Docker environment with minimal setup.
-- **Comprehensive Logging:** Detailed command output and error handling for safe operations.
 
-## 🛠 Requirements
+- **Docker volumes are always backed up at file level**
+- **SQL dumps are created only for explicitly defined databases**
 
-- **Linux Operating System** (with Docker installed) 🐧
+This results in the following behavior:
-- **Python 3.x** 🐍
-- **Docker & Docker Compose** 🔧
-- **rsync** installed on your system
 
-## 📥 Installation
+| Database defined | File backup | SQL dump |
+|------------------|-------------|----------|
+| No               | ✔ yes       | ✘ no     |
+| Yes              | ✔ yes       | ✔ yes    |
 
-You can install **Backup Docker Volumes to Local** easily via [Kevin's Package Manager](https://github.com/kevinveenbirkenbach/package-manager) using the alias **baudolo**:
+## 📁 Backup Layout
 
-```bash
+Backups are stored in a deterministic, fully nested structure:
-pkgmgr install baudolo
+
+```text
+<backups-dir>/
+└── <machine-hash>/
+    └── <repo-name>/
+        └── <timestamp>/
+            └── <volume-name>/
+                ├── files/
+                └── sql/
+                    └── <database>.backup.sql
 ```
 
-Alternatively, clone the repository directly:
+### Meaning of each level
+
+* `<machine-hash>`
+  SHA256 hash of `/etc/machine-id` (host separation)
+
+* `<repo-name>`
+  Logical backup namespace (project / stack)
+
+* `<timestamp>`
+  Backup generation (`YYYYMMDDHHMMSS`)
+
+* `<volume-name>`
+  Docker volume name
+
+* `files/`
+  Incremental file backup (rsync)
+
+* `sql/`
+  Optional SQL dumps (only for defined databases)
+
+## 🚀 Installation
+
+### Local (editable install)
 
 ```bash
-git clone https://github.com/kevinveenbirkenbach/backup-docker-to-local.git
+python3 -m venv .venv
-cd backup-docker-to-local
+source .venv/bin/activate
+pip install -e .
 ```
 
-## 🚀 Usage
+## 🌱 Database Definition (SQL Backup Scope)
 
-### Backup All Volumes
+### How SQL backups are defined
 
-To backup all Docker volumes, simply run:
+`baudolo` creates SQL dumps **only** for databases that are **explicitly defined**
+via configuration (e.g. a databases definition file or seeding step).
+
+If a database is **not defined**:
+
+* its Docker volume is still backed up (files)
+* **no SQL dump is created**
+
+> No database definition → file backup only
+> Database definition present → file backup + SQL dump
+
+### Why explicit definition?
+
+`baudolo` does **not** inspect running containers to guess databases.
+
+Databases must be explicitly defined to guarantee:
+
+* deterministic backups
+* predictable restore behavior
+* reproducible environments
+* zero accidental production data exposure
+
+### Required database metadata
+
+Each database definition provides:
+
+* database instance (container or logical instance)
+* database name
+* database user
+* database password
+
+This information is used by `baudolo` to execute
+`pg_dump`, `pg_dumpall`, or `mariadb-dump`.
+
+## 💾 Running a Backup
 
 ```bash
-./backup-docker-to-local.sh
+baudolo \
+  --compose-dir /srv/docker \
+  --databases-csv /etc/baudolo/databases.csv \
+  --database-containers central-postgres central-mariadb \
+  --images-no-stop-required alpine postgres mariadb mysql \
+  --images-no-backup-required redis busybox
 ```
 
-### Recovery
+### Common Backup Flags
 
-#### Recover Volume Files
+| Flag            | Description                                 |
+| --------------- | ------------------------------------------- |
+| `--everything`  | Always stop containers and re-run rsync     |
+| `--dump-only-sql`| Skip file backups only for DB volumes when dumps succeed; non-DB volumes are still backed up; fallback to files if no dump.    |
+| `--shutdown`    | Do not restart containers after backup      |
+| `--backups-dir` | Backup root directory (default: `/Backups`) |
+| `--repo-name`   | Backup namespace under machine hash         |
+
+## ♻️ Restore Operations
+
+### Restore Volume Files
 
 ```bash
-bash ./recover-docker-from-local.sh "{{volume_name}}" "$(sha256sum /etc/machine-id | head -c 64)" "{{version_to_recover}}"
+baudolo-restore files \
+  my-volume \
+  <machine-hash> \
+  <version> \
+  --backups-dir /Backups \
+  --repo-name my-repo
 ```
 
-#### Recover Database
+Restore into a **different target volume**:
-
-For example, to recover a MySQL/MariaDB database:
 
 ```bash
-docker exec -i mysql_container mysql -uroot -psecret database < db.sql
+baudolo-restore files \
+  target-volume \
+  <machine-hash> \
+  <version> \
+  --source-volume source-volume
 ```
 
-#### Debug Mode
+### Restore PostgreSQL
-
-To inspect what’s happening inside a container:
 
 ```bash
-docker run -it --entrypoint /bin/sh --rm --volumes-from {{container_name}} -v /Backups/:/Backups/ kevinveenbirkenbach/alpine-rsync
+baudolo-restore postgres \
+  my-volume \
+  <machine-hash> \
+  <version> \
+  --container postgres \
+  --db-name appdb \
+  --db-password secret \
+  --empty
 ```
 
+### Restore MariaDB / MySQL
+
+```bash
+baudolo-restore mariadb \
+  my-volume \
+  <machine-hash> \
+  <version> \
+  --container mariadb \
+  --db-name shopdb \
+  --db-password secret \
+  --empty
+```
+
+> `baudolo` automatically detects whether `mariadb` or `mysql`
+> is available inside the container
+
 ## 🔍 Backup Scheme
 
 The backup mechanism uses incremental backups with rsync and stamps directories with a unique hash. For more details on the backup scheme, check out [this blog post](https://blog.veen.world/blog/2020/12/26/how-i-backup-dedicated-root-servers/).  

Todo.md

@@ -1,2 +0,0 @@
-# Todo
-- Verify that restore backup is correct implemented

backup-docker-to-local.py

@@ -1,356 +0,0 @@
-#!/bin/python
-# Backups volumes of running containers
-
-import subprocess
-import os
-import re
-import pathlib
-import pandas
-from datetime import datetime
-import argparse
-
-class BackupException(Exception):
-    """Generic exception for backup errors."""
-    pass
-
-def execute_shell_command(command):
-    """Execute a shell command and return its output."""
-    print(command)
-    process = subprocess.Popen(
-        [command],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        shell=True
-    )
-    out, err = process.communicate()
-    if process.returncode != 0:
-        raise BackupException(
-            f"Error in command: {command}\n"
-            f"Output: {out}\nError: {err}\n"
-            f"Exit code: {process.returncode}"
-        )
-    return [line.decode("utf-8") for line in out.splitlines()]
-
-def create_version_directory():
-    """Create necessary directories for backup."""
-    version_dir = os.path.join(VERSIONS_DIR, BACKUP_TIME)
-    pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
-    return version_dir
-
-def get_machine_id():
-    """Get the machine identifier."""
-    return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
-
-### GLOBAL CONFIGURATION ###
-
-# Container names treated as special instances for database backups
-SPECIAL_INSTANCES = ['central-mariadb', 'central-postgres']
-
-# Images which do not require container stop for file backups
-IMAGES_NO_STOP_REQUIRED = []
-
-# Images to skip entirely
-IMAGES_NO_BACKUP_REQUIRED = [
-    'redis',
-    'memcached'
-]
-# Compose dirs requiring hard restart
-DOCKER_COMPOSE_HARD_RESTART_REQUIRED = ['mailu']
-
-# DEFINE CONSTANTS
-DIRNAME             = os.path.dirname(__file__)
-SCRIPTS_DIRECTORY   = pathlib.Path(os.path.realpath(__file__)).parent.parent
-DATABASES           = pandas.read_csv(os.path.join(DIRNAME, "databases.csv"), sep=";")
-REPOSITORY_NAME     = os.path.basename(DIRNAME)
-MACHINE_ID          = get_machine_id()
-BACKUPS_DIR         = '/Backups/'
-VERSIONS_DIR        = os.path.join(BACKUPS_DIR, MACHINE_ID, REPOSITORY_NAME)
-BACKUP_TIME         = datetime.now().strftime("%Y%m%d%H%M%S")
-VERSION_DIR         = create_version_directory()
-
-def get_instance(container):
-    """Extract the database instance name based on container name."""
-    if container in SPECIAL_INSTANCES:
-        instance_name = container
-    else:
-        instance_name = re.split("(_|-)(database|db|postgres)", container)[0]
-    print(f"Extracted instance name: {instance_name}")
-    return instance_name
-
-def stamp_directory():
-    """Stamp a directory using directory-validator."""
-    stamp_command = (
-        f"python {SCRIPTS_DIRECTORY}/directory-validator/"
-        f"directory-validator.py --stamp {VERSION_DIR}"
-    )
-    try:
-        execute_shell_command(stamp_command)
-        print(f"Successfully stamped directory: {VERSION_DIR}")
-    except BackupException as e:
-        print(f"Error stamping directory {VERSION_DIR}: {e}")
-        exit(1)
-
-def backup_database(container, volume_dir, db_type):
-    """Backup database (MariaDB or PostgreSQL) if applicable."""
-    print(f"Starting database backup for {container} using {db_type}...")
-    instance_name = get_instance(container)
-    database_entries = DATABASES.loc[DATABASES['instance'] == instance_name]
-    if database_entries.empty:
-        raise BackupException(f"No entry found for instance '{instance_name}'")
-    for database_entry in database_entries.iloc:
-        database_name     = database_entry['database']
-        database_username = database_entry['username']
-        database_password = database_entry['password']
-        backup_destination_dir = os.path.join(volume_dir, "sql")
-        pathlib.Path(backup_destination_dir).mkdir(parents=True, exist_ok=True)
-        backup_destination_file = os.path.join(
-            backup_destination_dir,
-            f"{database_name}.backup.sql"
-        )
-        if db_type == 'mariadb':
-            cmd = (
-                f"docker exec {container} "
-                f"/usr/bin/mariadb-dump -u {database_username} "
-                f"-p{database_password} {database_name} > {backup_destination_file}"
-            )
-            execute_shell_command(cmd)
-        if db_type == 'postgres':
-            cluster_file = os.path.join(
-                backup_destination_dir,
-                f"{instance_name}.cluster.backup.sql"
-            )
-            if not database_name:
-                fallback_pg_dumpall(
-                    container,
-                    database_username,
-                    database_password,
-                    cluster_file
-                )
-                return
-            try:
-                if database_password:
-                    cmd = (
-                        f"PGPASSWORD={database_password} docker exec -i {container} "
-                        f"pg_dump -U {database_username} -d {database_name} "
-                        f"-h localhost > {backup_destination_file}"
-                    )
-                else:
-                    cmd = (
-                        f"docker exec -i {container} pg_dump -U {database_username} "
-                        f"-d {database_name} -h localhost --no-password "
-                        f"> {backup_destination_file}"
-                    )
-                execute_shell_command(cmd)
-            except BackupException as e:
-                print(f"pg_dump failed: {e}")
-                print(f"Falling back to pg_dumpall for instance '{instance_name}'")
-                fallback_pg_dumpall(
-                    container,
-                    database_username,
-                    database_password,
-                    cluster_file
-                )
-        print(f"Database backup for database {container} completed.")
-
-def get_last_backup_dir(volume_name, current_backup_dir):
-    """Get the most recent backup directory for the specified volume."""
-    versions = sorted(os.listdir(VERSIONS_DIR), reverse=True)
-    for version in versions:
-        backup_dir = os.path.join(
-            VERSIONS_DIR, version, volume_name, "files", ""
-        )
-        if backup_dir != current_backup_dir and os.path.isdir(backup_dir):
-            return backup_dir
-    print(f"No previous backups available for volume: {volume_name}")
-    return None
-
-def getStoragePath(volume_name):
-    path = execute_shell_command(
-        f"docker volume inspect --format '{{{{ .Mountpoint }}}}' {volume_name}"
-    )[0]
-    return f"{path}/"
-
-def getFileRsyncDestinationPath(volume_dir):
-    path = os.path.join(volume_dir, "files")
-    return f"{path}/"
-
-def fallback_pg_dumpall(container, username, password, backup_destination_file):
-    """Fallback function to run pg_dumpall if pg_dump fails or no DB is defined."""
-    print(f"Running pg_dumpall for container '{container}'...")
-    cmd = (
-        f"PGPASSWORD={password} docker exec -i {container} "
-        f"pg_dumpall -U {username} -h localhost > {backup_destination_file}"
-    )
-    execute_shell_command(cmd)
-
-def backup_volume(volume_name, volume_dir):
-    """Perform incremental file backup of a Docker volume."""
-    try:
-        print(f"Starting backup routine for volume: {volume_name}")
-        dest = getFileRsyncDestinationPath(volume_dir)
-        pathlib.Path(dest).mkdir(parents=True, exist_ok=True)
-        last = get_last_backup_dir(volume_name, dest)
-        link_dest = f"--link-dest='{last}'" if last else ""
-        source = getStoragePath(volume_name)
-        cmd = (
-            f"rsync -abP --delete --delete-excluded "
-            f"{link_dest} {source} {dest}"
-        )
-        execute_shell_command(cmd)
-    except BackupException as e:
-        if "file has vanished" in str(e):
-            print("Warning: Some files vanished before transfer. Continuing.")
-        else:
-            raise
-    print(f"Backup routine for volume: {volume_name} completed.")
-
-def get_image_info(container):
-    return execute_shell_command(
-        f"docker inspect --format '{{{{.Config.Image}}}}' {container}"
-    )
-
-def has_image(container, image):
-    """Check if the container is using the image"""
-    info = get_image_info(container)[0]
-    return image in info
-
-def change_containers_status(containers, status):
-    """Stop or start a list of containers."""
-    if containers:
-        names = ' '.join(containers)
-        print(f"{status.capitalize()} containers: {names}...")
-        execute_shell_command(f"docker {status} {names}")
-    else:
-        print(f"No containers to {status}.")
-
-def is_image_whitelisted(container, images):
-    info =  (container)[0]
-    return any(img in info for img in images)
-
-def is_container_stop_required(containers):
-    """Check if any of the containers are using images that are not whitelisted."""
-    return any(
-        not is_image_whitelisted(c, IMAGES_NO_STOP_REQUIRED)
-        for c in containers
-    )
-
-def create_volume_directory(volume_name):
-    """Create necessary directories for backup."""
-    path = os.path.join(VERSION_DIR, volume_name)
-    pathlib.Path(path).mkdir(parents=True, exist_ok=True)
-    return path
-
-def is_image_ignored(container):
-    """Check if the container's image is one of the ignored images."""
-    return any(has_image(container, img) for img in IMAGES_NO_BACKUP_REQUIRED)
-
-def backup_with_containers_paused(volume_name, volume_dir, containers, shutdown):
-    change_containers_status(containers, 'stop')
-    backup_volume(volume_name, volume_dir)
-    if not shutdown:
-        change_containers_status(containers, 'start')
-
-def backup_mariadb_or_postgres(container, volume_dir):
-    """Performs database image specific backup procedures"""
-    for img in ['mariadb', 'postgres']:
-        if has_image(container, img):
-            backup_database(container, volume_dir, img)
-            return True
-    return False
-
-def default_backup_routine_for_volume(volume_name, containers, shutdown):
-    """Perform backup routine for a given volume."""
-    vol_dir = ""
-    for c in containers:
-        if is_image_ignored(c):
-            print(f"Ignoring volume '{volume_name}' linked to container '{c}'.")
-            continue
-        vol_dir = create_volume_directory(volume_name)
-        if backup_mariadb_or_postgres(c, vol_dir):
-            return
-    if vol_dir:
-        backup_volume(volume_name, vol_dir)
-        if is_container_stop_required(containers):
-            backup_with_containers_paused(volume_name, vol_dir, containers, shutdown)
-
-def backup_everything(volume_name, containers, shutdown):
-    """Perform file backup routine for a given volume."""
-    vol_dir = create_volume_directory(volume_name)
-    for c in containers:
-        backup_mariadb_or_postgres(c, vol_dir)
-    backup_volume(volume_name, vol_dir)
-    backup_with_containers_paused(volume_name, vol_dir, containers, shutdown)
-
-def hard_restart_docker_services(dir_path):
-    """Perform a hard restart of docker-compose services in the given directory."""
-    try:
-        print(f"Performing hard restart for docker-compose services in: {dir_path}")
-        subprocess.run(["docker-compose", "down"], cwd=dir_path, check=True)
-        subprocess.run(["docker-compose", "up", "-d"], cwd=dir_path, check=True)
-        print(f"Hard restart completed successfully in: {dir_path}")
-    except subprocess.CalledProcessError as e:
-        print(f"Error during hard restart in {dir_path}: {e}")
-        exit(2)
-
-def handle_docker_compose_services(parent_directory):
-    """Iterate through directories and restart or hard restart services as needed."""
-    for entry in os.scandir(parent_directory):
-        if entry.is_dir():
-            dir_path = entry.path
-            name = os.path.basename(dir_path)
-            print(f"Checking directory: {dir_path}")
-            compose_file = os.path.join(dir_path, "docker-compose.yml")
-            if os.path.isfile(compose_file):
-                print(f"Found docker-compose.yml in {dir_path}.")
-                if name in DOCKER_COMPOSE_HARD_RESTART_REQUIRED:
-                    print(f"Directory {name} detected. Performing hard restart...")
-                    hard_restart_docker_services(dir_path)
-                else:
-                    print(f"No restart required for services in {dir_path}...")
-            else:
-                print(f"No docker-compose.yml found in {dir_path}. Skipping.")
-
-def main():
-    global SPECIAL_INSTANCES, IMAGES_NO_STOP_REQUIRED
-    parser = argparse.ArgumentParser(description='Backup Docker volumes.')
-    parser.add_argument('--everything', action='store_true',
-                        help='Force file backup for all volumes and additional execute database dumps')
-    parser.add_argument('--shutdown', action='store_true',
-                        help='Doesn\'t restart containers after backup')
-    parser.add_argument('--compose-dir', type=str, required=True,
-                        help='Path to the parent directory containing docker-compose setups')
-    parser.add_argument(
-        '--special-instances',
-        nargs='+',
-        default=SPECIAL_INSTANCES,
-        help='List of container names treated as special instances for database backups'
-    )
-    parser.add_argument(
-        '--images-no-stop-required',
-        nargs='+',
-        help='List of image names for which containers should not be stopped during file backup'
-    )
-    args = parser.parse_args()
-    SPECIAL_INSTANCES = args.special_instances
-    IMAGES_NO_STOP_REQUIRED = args.images_no_stop_required
-
-    print('Start volume backups...')
-    volume_names = execute_shell_command("docker volume ls --format '{{.Name}}'")
-    for volume_name in volume_names:
-        print(f'Start backup routine for volume: {volume_name}')
-        containers = execute_shell_command(
-            f"docker ps --filter volume=\"{volume_name}\" --format '{{{{.Names}}}}'"
-        )
-        if args.everything:
-            backup_everything(volume_name, containers, args.shutdown)
-        else:
-            default_backup_routine_for_volume(volume_name, containers, args.shutdown)
-
-    stamp_directory()
-    print('Finished volume backups.')
-
-    print('Handling Docker Compose services...')
-    handle_docker_compose_services(args.compose_dir)
-
-if __name__ == "__main__":
-    main()

database_entry_seeder.py

@@ -1,50 +0,0 @@
-import pandas as pd
-import argparse
-import os
-
-def check_and_add_entry(file_path, instance, database, username, password):
-    # Check if the file exists and is not empty
-    if os.path.exists(file_path) and os.path.getsize(file_path) > 0:
-        # Read the existing CSV file with header
-        df = pd.read_csv(file_path, sep=';')
-    else:
-        # Create a new DataFrame with columns if file does not exist
-        df = pd.DataFrame(columns=['instance', 'database', 'username', 'password'])
-
-    # Check if the entry exists and remove it
-    mask = (
-        (df['instance'] == instance) &
-        ((df['database'] == database) |
-        (((df['database'].isna()) | (df['database'] == '')) & (database == ''))) &
-        (df['username'] == username)
-    )
-
-    if not df[mask].empty:
-        print("Replacing existing entry.")
-        df = df[~mask]
-    else:
-        print("Adding new entry.")
-
-    # Create a new DataFrame for the new entry
-    new_entry = pd.DataFrame([{'instance': instance, 'database': database, 'username': username, 'password': password}])
-
-    # Add (or replace) the entry using concat
-    df = pd.concat([df, new_entry], ignore_index=True)
-
-    # Save the updated CSV file
-    df.to_csv(file_path, sep=';', index=False)
-
-def main():
-    parser = argparse.ArgumentParser(description="Check and replace (or add) a database entry in a CSV file.")
-    parser.add_argument("file_path", help="Path to the CSV file")
-    parser.add_argument("instance", help="Database instance")
-    parser.add_argument("database", help="Database name")
-    parser.add_argument("username", help="Username")
-    parser.add_argument("password", nargs='?', default="", help="Password (optional)")
-
-    args = parser.parse_args()
-
-    check_and_add_entry(args.file_path, args.instance, args.database, args.username, args.password)
-
-if __name__ == "__main__":
-    main()

databases.csv.tpl

@@ -1 +0,0 @@
-database;username;password;container

pyproject.toml

@@ -0,0 +1,29 @@
+[build-system]
+requires = ["setuptools>=69", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "backup-docker-to-local"
+version = "1.2.0"
+description = "Backup Docker volumes to local with rsync and optional DB dumps."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "AGPL-3.0-or-later" }
+authors = [{ name = "Kevin Veen-Birkenbach" }]
+
+dependencies = [
+  "pandas",
+  "dirval",
+]
+
+[project.scripts]
+baudolo = "baudolo.backup.__main__:main"
+baudolo-restore = "baudolo.restore.__main__:main"
+baudolo-seed = "baudolo.seed.__main__:main"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+exclude = ["tests*"]

recover-docker-from-local.sh

@@ -1,85 +0,0 @@
-#!/bin/bash
-
-# Check minimum number of arguments
-if [ $# -lt 3 ]; then
-  echo "ERROR: Not enough arguments. Please provide at least a volume name, backup hash, and version."
-  exit 1
-fi
-
-volume_name="$1"                # Volume-Name
-backup_hash="$2"                # Hashed Machine ID
-version="$3"                    # version to recover
-
-# DATABASE PARAMETERS
-database_type="$4"              # Valid values; mariadb, postgress
-database_container="$5"         # optional
-database_password="$6"          # optional
-database_name="$7"              # optional
-database_user="$database_name"  
-
-
-backup_folder="Backups/$backup_hash/backup-docker-to-local/$version/$volume_name"
-backup_files="/$backup_folder/files"
-backup_sql="/$backup_folder/sql/$database_name.backup.sql"
-
-# DATABASE RECOVERY
-
-if [ ! -z "$database_type" ]; then
-  if [ "$database_type" = "postgres" ]; then
-    if [ -n "$database_container" ] && [ -n "$database_password" ] && [ -n "$database_name" ]; then
-      echo "Recover PostgreSQL dump"
-      export PGPASSWORD="$database_password"
-      cat "$backup_sql" | docker exec -i "$database_container" psql -v ON_ERROR_STOP=1 -U "$database_user" -d "$database_name"
-      if [ $? -ne 0 ]; then
-          echo "ERROR: Failed to recover PostgreSQL dump"
-          exit 1
-      fi
-      exit 0
-    fi
-  elif [ "$database_type" = "mariadb" ]; then
-    if [ -n "$database_container" ] && [ -n "$database_password" ] && [ -n "$database_name" ]; then
-      echo "recover mysql dump"
-      cat "$backup_sql" | docker exec -i "$database_container" mariadb -u "$database_user" --password="$database_password" "$database_name"
-      if [ $? -ne 0 ]; then
-          echo "ERROR: Failed to recover mysql dump"
-          exit 1
-      fi
-      exit 0
-    fi
-  fi
-  echo "A database backup exists, but a parameter is missing."
-  exit 1
-fi
-
-# FILE RECOVERY
-
-echo "Inspect volume $volume_name"
-docker volume inspect "$volume_name"
-exit_status_volume_inspect=$?
-
-if [ $exit_status_volume_inspect -eq 0 ]; then
-    echo "Volume $volume_name already exists"
-else
-    echo "Create volume $volume_name"
-    docker volume create "$volume_name"
-    if [ $? -ne 0 ]; then
-        echo "ERROR: Failed to create volume $volume_name"
-        exit 1
-    fi
-fi
-
-if [ -d "$backup_files" ]; then    
-  echo "recover files"
-  docker run --rm -v "$volume_name:/recover/" -v "$backup_files:/backup/" "kevinveenbirkenbach/alpine-rsync" sh -c "rsync -avv --delete /backup/ /recover/"
-  if [ $? -ne 0 ]; then
-    echo "ERROR: Failed to recover files"
-    exit 1
-  fi
-  exit 0
-else
-  echo "ERROR: $backup_files doesn't exist"
-  exit 1
-fi
-
-echo "ERROR: Unhandled case"
-exit 1

requirements.yml

@@ -1,3 +0,0 @@
-pacman:
-  - lsof
-  - python-pandas

restore_backup.py

@@ -1,170 +0,0 @@
-#!/usr/bin/env python3
-# @todo Not tested yet. Needs to be tested
-"""
-restore_backup.py
-
-A script to recover Docker volumes and database dumps from local backups.
-Supports an --empty flag to clear the database objects before import (drops all tables/functions etc.).
-"""
-import argparse
-import os
-import sys
-import subprocess
-
-
-def run_command(cmd, capture_output=False, input=None, **kwargs):
-    """Run a subprocess command and handle errors."""
-    try:
-        result = subprocess.run(cmd, check=True, capture_output=capture_output, input=input, **kwargs)
-        return result
-    except subprocess.CalledProcessError as e:
-        print(f"ERROR: Command '{' '.join(cmd)}' failed with exit code {e.returncode}")
-        if e.stdout:
-            print(e.stdout.decode())
-        if e.stderr:
-            print(e.stderr.decode())
-        sys.exit(1)
-
-
-def recover_postgres(container, password, db_name, user, backup_sql, empty=False):
-    print("Recovering PostgreSQL dump...")
-    os.environ['PGPASSWORD'] = password
-    if empty:
-        print("Dropping existing PostgreSQL objects...")
-        # Drop all tables, views, sequences, functions in public schema
-        drop_sql = """
-DO $$ DECLARE r RECORD;
-BEGIN
-  FOR r IN (
-    SELECT table_name AS name, 'TABLE' AS type FROM information_schema.tables WHERE table_schema='public'
-    UNION ALL
-    SELECT routine_name AS name, 'FUNCTION' AS type FROM information_schema.routines WHERE specific_schema='public'
-    UNION ALL
-    SELECT sequence_name AS name, 'SEQUENCE' AS type FROM information_schema.sequences WHERE sequence_schema='public'
-  ) LOOP
-    -- Use %s for type to avoid quoting the SQL keyword
-    EXECUTE format('DROP %s public.%I CASCADE', r.type, r.name);
-  END LOOP;
-END
-$$;
-"""
-        run_command([
-            'docker', 'exec', '-i', container,
-            'psql', '-v', 'ON_ERROR_STOP=1', '-U', user, '-d', db_name
-        ], input=drop_sql.encode())
-        print("Existing objects dropped.")
-    print("Importing the dump...")
-    with open(backup_sql, 'rb') as f:
-        run_command([
-            'docker', 'exec', '-i', container,
-            'psql', '-v', 'ON_ERROR_STOP=1', '-U', user, '-d', db_name
-        ], stdin=f)
-    print("PostgreSQL recovery complete.")
-
-
-def recover_mariadb(container, password, db_name, user, backup_sql, empty=False):
-    print("Recovering MariaDB dump...")
-    if empty:
-        print("Dropping existing MariaDB tables...")
-        # Disable foreign key checks
-        run_command([
-            'docker', 'exec', container,
-            'mysql', '-u', user, f"--password={password}", '-e', 'SET FOREIGN_KEY_CHECKS=0;'
-        ])
-        # Get all table names
-        result = run_command([
-            'docker', 'exec', container,
-            'mysql', '-u', user, f"--password={password}", '-N', '-e',
-            f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{db_name}';"
-        ], capture_output=True)
-        tables = result.stdout.decode().split()
-        for tbl in tables:
-            run_command([
-                'docker', 'exec', container,
-                'mysql', '-u', user, f"--password={password}", '-e',
-                f"DROP TABLE IF EXISTS `{db_name}`.`{tbl}`;"
-            ])
-        # Enable foreign key checks
-        run_command([
-            'docker', 'exec', container,
-            'mysql', '-u', user, f"--password={password}", '-e', 'SET FOREIGN_KEY_CHECKS=1;'
-        ])
-        print("Existing tables dropped.")
-    print("Importing the dump...")
-    with open(backup_sql, 'rb') as f:
-        run_command([
-            'docker', 'exec', '-i', container,
-            'mariadb', '-u', user, f"--password={password}", db_name
-        ], stdin=f)
-    print("MariaDB recovery complete.")
-
-
-def recover_files(volume_name, backup_files):
-    print(f"Inspecting volume {volume_name}...")
-    inspect = subprocess.run(['docker', 'volume', 'inspect', volume_name], stdout=subprocess.DEVNULL)
-    if inspect.returncode != 0:
-        print(f"Volume {volume_name} does not exist. Creating...")
-        run_command(['docker', 'volume', 'create', volume_name])
-    else:
-        print(f"Volume {volume_name} already exists.")
-
-    if not os.path.isdir(backup_files):
-        print(f"ERROR: Backup files folder '{backup_files}' does not exist.")
-        sys.exit(1)
-
-    print("Recovering files...")
-    run_command([
-        'docker', 'run', '--rm',
-        '-v', f"{volume_name}:/recover/",
-        '-v', f"{backup_files}:/backup/",
-        'kevinveenbirkenbach/alpine-rsync',
-        'sh', '-c', 'rsync -avv --delete /backup/ /recover/'
-    ])
-    print("File recovery complete.")
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description='Recover Docker volumes and database dumps from local backups.'
-    )
-    parser.add_argument('volume_name', help='Name of the Docker volume')
-    parser.add_argument('backup_hash', help='Hashed Machine ID')
-    parser.add_argument('version', help='Version to recover')
-
-    parser.add_argument('--db-type', choices=['postgres', 'mariadb'], help='Type of database backup')
-    parser.add_argument('--db-container', help='Docker container name for the database')
-    parser.add_argument('--db-password', help='Password for the database user')
-    parser.add_argument('--db-name', help='Name of the database')
-    parser.add_argument('--empty', action='store_true', help='Drop existing database objects before importing')
-
-    args = parser.parse_args()
-
-    volume = args.volume_name
-    backup_hash = args.backup_hash
-    version = args.version
-
-    backup_folder = os.path.join('Backups', backup_hash, 'backup-docker-to-local', version, volume)
-    backup_files = os.path.join(os.sep, backup_folder, 'files')
-    backup_sql = None
-    if args.db_name:
-        backup_sql = os.path.join(os.sep, backup_folder, 'sql', f"{args.db_name}.backup.sql")
-
-    # Database recovery
-    if args.db_type:
-        if not (args.db_container and args.db_password and args.db_name):
-            print("ERROR: A database backup exists, aber ein Parameter fehlt.")
-            sys.exit(1)
-
-        user = args.db_name
-        if args.db_type == 'postgres':
-            recover_postgres(args.db_container, args.db_password, args.db_name, user, backup_sql, empty=args.empty)
-        else:
-            recover_mariadb(args.db_container, args.db_password, args.db_name, user, backup_sql, empty=args.empty)
-        sys.exit(0)
-
-    # File recovery
-    recover_files(volume, backup_files)
-
-
-if __name__ == '__main__':
-    main()

restore_postgres_databases.py

@@ -1,96 +0,0 @@
-#!/usr/bin/env python3
-"""
-Restore multiple PostgreSQL databases from .backup.sql files via a Docker container.
-
-Usage:
-  ./restore_databases.py /path/to/backup_dir container_name
-"""
-import argparse
-import subprocess
-import sys
-import os
-import glob
-
-def run_command(cmd, stdin=None):
-    """
-    Run a subprocess command and abort immediately on any failure.
-    :param cmd: list of command parts
-    :param stdin: file-like object to use as stdin
-    """
-    subprocess.run(cmd, stdin=stdin, check=True)
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Restore Postgres databases from backup SQL files via Docker container."
-    )
-    parser.add_argument(
-        "backup_dir",
-        help="Path to directory containing .backup.sql files"
-    )
-    parser.add_argument(
-        "container",
-        help="Name of the Postgres Docker container"
-    )
-    args = parser.parse_args()
-
-    backup_dir = args.backup_dir
-    container = args.container
-
-    pattern = os.path.join(backup_dir, "*.backup.sql")
-    sql_files = sorted(glob.glob(pattern))
-    if not sql_files:
-        print(f"No .backup.sql files found in {backup_dir}", file=sys.stderr)
-        sys.exit(1)
-
-    for sqlfile in sql_files:
-        # Extract database name by stripping the full suffix '.backup.sql'
-        filename = os.path.basename(sqlfile)
-        if not filename.endswith('.backup.sql'):
-            continue
-        dbname = filename[:-len('.backup.sql')]
-        print(f"=== Processing {sqlfile} → database: {dbname} ===")
-
-        # Drop the database, forcing disconnect of sessions if necessary
-        run_command([
-            "docker", "exec", "-i", container,
-            "psql", "-U", "postgres", "-c",
-            f"DROP DATABASE IF EXISTS \"{dbname}\" WITH (FORCE);"
-        ])
-
-        # Create a fresh database
-        run_command([
-            "docker", "exec", "-i", container,
-            "psql", "-U", "postgres", "-c",
-            f"CREATE DATABASE \"{dbname}\";"
-        ])
-
-        # Ensure the ownership role exists
-        print(f"Ensuring role '{dbname}' exists...")
-        run_command([
-            "docker", "exec", "-i", container,
-            "psql", "-U", "postgres", "-c",
-            (
-                "DO $$BEGIN "
-                f"IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '{dbname}') THEN "
-                f"CREATE ROLE \"{dbname}\"; "
-                "END IF; "
-                "END$$;"
-            )
-        ])
-
-        # Restore the dump into the database by streaming file (will abort on first error)
-        print(f"Restoring dump into {dbname} (this may take a while)…")
-        with open(sqlfile, 'rb') as infile:
-            run_command([
-                "docker", "exec", "-i", container,
-                "psql", "-U", "postgres", "-d", dbname
-            ], stdin=infile)
-
-        print(f"✔ {dbname} restored.")
-
-    print("All databases have been restored.")
-
-
-if __name__ == "__main__":
-    main()

scripts/test-e2e.sh

@@ -0,0 +1,234 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# -----------------------------------------------------------------------------
+# E2E runner using Docker-in-Docker (DinD) with debug-on-failure
+#
+# Debug toggles:
+#   E2E_KEEP_ON_FAIL=1   -> keep DinD + volumes + network if tests fail
+#   E2E_KEEP_VOLUMES=1   -> keep volumes even on success/cleanup
+#   E2E_DEBUG_SHELL=1    -> open an interactive shell in the test container instead of running tests
+#   E2E_ARTIFACTS_DIR=./artifacts
+# -----------------------------------------------------------------------------
+
+NET="${E2E_NET:-baudolo-e2e-net}"
+DIND="${E2E_DIND_NAME:-baudolo-e2e-dind}"
+DIND_VOL="${E2E_DIND_VOL:-baudolo-e2e-dind-data}"
+E2E_TMP_VOL="${E2E_TMP_VOL:-baudolo-e2e-tmp}"
+
+DIND_HOST="${E2E_DIND_HOST:-tcp://127.0.0.1:2375}"
+DIND_HOST_IN_NET="${E2E_DIND_HOST_IN_NET:-tcp://${DIND}:2375}"
+
+IMG="${E2E_IMAGE:-baudolo:local}"
+RSYNC_IMG="${E2E_RSYNC_IMAGE:-ghcr.io/kevinveenbirkenbach/alpine-rsync}"
+
+READY_TIMEOUT_SECONDS="${E2E_READY_TIMEOUT_SECONDS:-120}"
+ARTIFACTS_DIR="${E2E_ARTIFACTS_DIR:-./artifacts}"
+
+KEEP_ON_FAIL="${E2E_KEEP_ON_FAIL:-0}"
+KEEP_VOLUMES="${E2E_KEEP_VOLUMES:-0}"
+DEBUG_SHELL="${E2E_DEBUG_SHELL:-0}"
+
+FAILED=0
+TS="$(date +%Y%m%d%H%M%S)"
+
+mkdir -p "${ARTIFACTS_DIR}"
+
+log() { echo ">> $*"; }
+
+dump_debug() {
+  log "DEBUG: collecting diagnostics into ${ARTIFACTS_DIR}"
+
+  {
+    echo "=== Host docker version ==="
+    docker version || true
+    echo
+    echo "=== Host docker info ==="
+    docker info || true
+    echo
+    echo "=== DinD reachable? (docker -H ${DIND_HOST} version) ==="
+    docker -H "${DIND_HOST}" version || true
+    echo
+  } > "${ARTIFACTS_DIR}/debug-host-${TS}.txt" 2>&1 || true
+
+  # DinD logs
+  docker logs --tail=5000 "${DIND}" > "${ARTIFACTS_DIR}/dind-logs-${TS}.txt" 2>&1 || true
+
+  # DinD state
+  {
+    echo "=== docker -H ps -a ==="
+    docker -H "${DIND_HOST}" ps -a || true
+    echo
+    echo "=== docker -H images ==="
+    docker -H "${DIND_HOST}" images || true
+    echo
+    echo "=== docker -H network ls ==="
+    docker -H "${DIND_HOST}" network ls || true
+    echo
+    echo "=== docker -H volume ls ==="
+    docker -H "${DIND_HOST}" volume ls || true
+    echo
+    echo "=== docker -H system df ==="
+    docker -H "${DIND_HOST}" system df || true
+  } > "${ARTIFACTS_DIR}/debug-dind-${TS}.txt" 2>&1 || true
+
+  # Try to capture recent events (best effort; might be noisy)
+  docker -H "${DIND_HOST}" events --since 10m --until 0s \
+    > "${ARTIFACTS_DIR}/dind-events-${TS}.txt" 2>&1 || true
+
+  # Dump shared /tmp content from the tmp volume:
+  # We create a temporary container that mounts the volume, then tar its content.
+  # (Does not rely on host filesystem paths.)
+  log "DEBUG: archiving shared /tmp (volume ${E2E_TMP_VOL})"
+  docker -H "${DIND_HOST}" run --rm \
+    -v "${E2E_TMP_VOL}:/tmp" \
+    alpine:3.20 \
+    sh -lc 'cd /tmp && tar -czf /out.tar.gz . || true' \
+    >/dev/null 2>&1 || true
+
+  # The above writes inside the container FS, not to host. So do it properly:
+  # Use "docker cp" from a temp container.
+  local tmpc="baudolo-e2e-tmpdump-${TS}"
+  docker -H "${DIND_HOST}" rm -f "${tmpc}" >/dev/null 2>&1 || true
+  docker -H "${DIND_HOST}" create --name "${tmpc}" -v "${E2E_TMP_VOL}:/tmp" alpine:3.20 \
+    sh -lc 'cd /tmp && tar -czf /tmpdump.tar.gz . || true' >/dev/null
+  docker -H "${DIND_HOST}" start -a "${tmpc}" >/dev/null 2>&1 || true
+  docker -H "${DIND_HOST}" cp "${tmpc}:/tmpdump.tar.gz" "${ARTIFACTS_DIR}/e2e-tmp-${TS}.tar.gz" >/dev/null 2>&1 || true
+  docker -H "${DIND_HOST}" rm -f "${tmpc}" >/dev/null 2>&1 || true
+
+  log "DEBUG: artifacts written:"
+  ls -la "${ARTIFACTS_DIR}" | sed 's/^/   /' || true
+}
+
+cleanup() {
+  if [ "${FAILED}" -eq 1 ] && [ "${KEEP_ON_FAIL}" = "1" ]; then
+    log "KEEP_ON_FAIL=1 and failure detected -> skipping cleanup."
+    log "Next steps:"
+    echo "   - Inspect DinD logs: docker logs ${DIND} | less"
+    echo "   - Use DinD daemon:  docker -H ${DIND_HOST} ps -a"
+    echo "   - Shared tmp vol:   docker -H ${DIND_HOST} run --rm -v ${E2E_TMP_VOL}:/tmp alpine:3.20 ls -la /tmp"
+    echo "   - DinD docker root: docker -H ${DIND_HOST} run --rm -v ${DIND_VOL}:/var/lib/docker alpine:3.20 ls -la /var/lib/docker/volumes"
+    return 0
+  fi
+
+  log "Cleanup: stopping ${DIND} and removing network ${NET}"
+  docker rm -f "${DIND}" >/dev/null 2>&1 || true
+  docker network rm "${NET}" >/dev/null 2>&1 || true
+
+  if [ "${KEEP_VOLUMES}" != "1" ]; then
+    docker volume rm -f "${DIND_VOL}" >/dev/null 2>&1 || true
+    docker volume rm -f "${E2E_TMP_VOL}" >/dev/null 2>&1 || true
+  else
+    log "Keeping volumes (E2E_KEEP_VOLUMES=1): ${DIND_VOL}, ${E2E_TMP_VOL}"
+  fi
+}
+trap cleanup EXIT INT TERM
+
+log "Creating network ${NET} (if missing)"
+docker network inspect "${NET}" >/dev/null 2>&1 || docker network create "${NET}" >/dev/null
+
+log "Removing old ${DIND} (if any)"
+docker rm -f "${DIND}" >/dev/null 2>&1 || true
+
+log "(Re)creating DinD data volume ${DIND_VOL}"
+docker volume rm -f "${DIND_VOL}" >/dev/null 2>&1 || true
+docker volume create "${DIND_VOL}" >/dev/null
+
+log "(Re)creating shared /tmp volume ${E2E_TMP_VOL}"
+docker volume rm -f "${E2E_TMP_VOL}" >/dev/null 2>&1 || true
+docker volume create "${E2E_TMP_VOL}" >/dev/null
+
+log "Starting Docker-in-Docker daemon ${DIND}"
+docker run -d --privileged \
+  --name "${DIND}" \
+  --network "${NET}" \
+  -e DOCKER_TLS_CERTDIR="" \
+  -v "${DIND_VOL}:/var/lib/docker" \
+  -v "${E2E_TMP_VOL}:/tmp" \
+  -p 2375:2375 \
+  docker:dind \
+  --host=tcp://0.0.0.0:2375 \
+  --tls=false >/dev/null
+
+log "Waiting for DinD to be ready..."
+for i in $(seq 1 "${READY_TIMEOUT_SECONDS}"); do
+  if docker -H "${DIND_HOST}" version >/dev/null 2>&1; then
+    log "DinD is ready."
+    break
+  fi
+  sleep 1
+  if [ "${i}" -eq "${READY_TIMEOUT_SECONDS}" ]; then
+    echo "ERROR: DinD did not become ready in time"
+    docker logs --tail=200 "${DIND}" || true
+    FAILED=1
+    dump_debug || true
+    exit 1
+  fi
+done
+
+log "Pre-pulling helper images in DinD..."
+log " - Pulling: ${RSYNC_IMG}"
+docker -H "${DIND_HOST}" pull "${RSYNC_IMG}"
+
+log "Ensuring alpine exists in DinD (for debug helpers)"
+docker -H "${DIND_HOST}" pull alpine:3.20 >/dev/null
+
+log "Loading ${IMG} image into DinD..."
+docker save "${IMG}" | docker -H "${DIND_HOST}" load >/dev/null
+
+log "Running E2E tests inside DinD"
+set +e
+if [ "${DEBUG_SHELL}" = "1" ]; then
+  log "E2E_DEBUG_SHELL=1 -> opening shell in test container"
+  docker run --rm -it \
+    --network "${NET}" \
+    -e DOCKER_HOST="${DIND_HOST_IN_NET}" \
+    -e E2E_RSYNC_IMAGE="${RSYNC_IMG}" \
+    -v "${DIND_VOL}:/var/lib/docker:ro" \
+    -v "${E2E_TMP_VOL}:/tmp" \
+    "${IMG}" \
+    sh -lc '
+      set -e
+      if [ ! -f /etc/machine-id ]; then
+        mkdir -p /etc
+        cat /proc/sys/kernel/random/uuid > /etc/machine-id
+      fi
+      echo ">> DOCKER_HOST=${DOCKER_HOST}"
+      docker ps -a || true
+      exec sh
+    '
+  rc=$?
+else
+  docker run --rm \
+    --network "${NET}" \
+    -e DOCKER_HOST="${DIND_HOST_IN_NET}" \
+    -e E2E_RSYNC_IMAGE="${RSYNC_IMG}" \
+    -v "${DIND_VOL}:/var/lib/docker:ro" \
+    -v "${E2E_TMP_VOL}:/tmp" \
+    "${IMG}" \
+    sh -lc '
+    set -euo pipefail
+    set -x
+    export PYTHONUNBUFFERED=1
+
+    export TMPDIR=/tmp TMP=/tmp TEMP=/tmp
+
+    if [ ! -f /etc/machine-id ]; then
+        mkdir -p /etc
+        cat /proc/sys/kernel/random/uuid > /etc/machine-id
+    fi
+
+    python -m unittest discover -t . -s tests/e2e -p "test_*.py" -v -f
+    '
+  rc=$?
+fi
+set -e
+
+if [ "${rc}" -ne 0 ]; then
+  FAILED=1
+  echo "ERROR: E2E tests failed (exit code: ${rc})"
+  dump_debug || true
+  exit "${rc}"
+fi
+
+log "E2E tests passed."

src/baudolo/backup/__init__.py

@@ -0,0 +1 @@
+"""Baudolo backup package."""

src/baudolo/backup/__main__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+from .app import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/baudolo/backup/app.py

@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+import os
+import pathlib
+from datetime import datetime
+
+import pandas
+from dirval import create_stamp_file
+
+from .cli import parse_args
+from .compose import handle_docker_compose_services
+from .db import backup_database
+from .docker import (
+    change_containers_status,
+    containers_using_volume,
+    docker_volume_names,
+    get_image_info,
+    has_image,
+)
+from .shell import execute_shell_command
+from .volume import backup_volume
+
+
+def get_machine_id() -> str:
+    return execute_shell_command("sha256sum /etc/machine-id")[0][0:64]
+
+
+def stamp_directory(version_dir: str) -> None:
+    """
+    Use dirval as a Python library to stamp the directory (no CLI dependency).
+    """
+    create_stamp_file(version_dir)
+
+
+def create_version_directory(versions_dir: str, backup_time: str) -> str:
+    version_dir = os.path.join(versions_dir, backup_time)
+    pathlib.Path(version_dir).mkdir(parents=True, exist_ok=True)
+    return version_dir
+
+
+def create_volume_directory(version_dir: str, volume_name: str) -> str:
+    path = os.path.join(version_dir, volume_name)
+    pathlib.Path(path).mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def is_image_ignored(container: str, images_no_backup_required: list[str]) -> bool:
+    if not images_no_backup_required:
+        return False
+    img = get_image_info(container)
+    return any(pat in img for pat in images_no_backup_required)
+
+
+def volume_is_fully_ignored(
+    containers: list[str], images_no_backup_required: list[str]
+) -> bool:
+    """
+    Skip file backup only if all containers linked to the volume are ignored.
+    """
+    if not containers:
+        return False
+    return all(is_image_ignored(c, images_no_backup_required) for c in containers)
+
+
+def requires_stop(containers: list[str], images_no_stop_required: list[str]) -> bool:
+    """
+    Stop is required if ANY container image is NOT in the whitelist patterns.
+    """
+    for c in containers:
+        img = get_image_info(c)
+        if not any(pat in img for pat in images_no_stop_required):
+            return True
+    return False
+
+def backup_mariadb_or_postgres(
+    *,
+    container: str,
+    volume_dir: str,
+    databases_df: "pandas.DataFrame",
+    database_containers: list[str],
+) -> tuple[bool, bool]:
+    """
+    Returns (is_db_container, dumped_any)
+    """
+    for img in ["mariadb", "postgres"]:
+        if has_image(container, img):
+            dumped = backup_database(
+                container=container,
+                volume_dir=volume_dir,
+                db_type=img,
+                databases_df=databases_df,
+                database_containers=database_containers,
+            )
+            return True, dumped
+    return False, False
+
+
+def _backup_dumps_for_volume(
+    *,
+    containers: list[str],
+    vol_dir: str,
+    databases_df: "pandas.DataFrame",
+    database_containers: list[str],
+) -> tuple[bool, bool]:
+    """
+    Returns (found_db_container, dumped_any)
+    """
+    found_db = False
+    dumped_any = False
+
+    for c in containers:
+        is_db, dumped = backup_mariadb_or_postgres(
+            container=c,
+            volume_dir=vol_dir,
+            databases_df=databases_df,
+            database_containers=database_containers,
+        )
+        if is_db:
+            found_db = True
+        if dumped:
+            dumped_any = True
+
+    return found_db, dumped_any
+
+
+def main() -> int:
+    args = parse_args()
+
+    machine_id = get_machine_id()
+    backup_time = datetime.now().strftime("%Y%m%d%H%M%S")
+
+    versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
+    version_dir = create_version_directory(versions_dir, backup_time)
+
+    # IMPORTANT:
+    # - keep_default_na=False prevents empty fields from turning into NaN
+    # - dtype=str keeps all columns stable for comparisons/validation
+    databases_df = pandas.read_csv(
+        args.databases_csv, sep=";", keep_default_na=False, dtype=str
+    )
+
+    print("💾 Start volume backups...", flush=True)
+
+    for volume_name in docker_volume_names():
+        print(f"Start backup routine for volume: {volume_name}", flush=True)
+        containers = containers_using_volume(volume_name)
+
+        # EARLY SKIP: if all linked containers are ignored, do not create any dirs
+        if volume_is_fully_ignored(containers, args.images_no_backup_required):
+            print(
+                f"Skipping volume '{volume_name}' entirely (all linked containers are ignored).",
+                flush=True,
+            )
+            continue
+
+        vol_dir = create_volume_directory(version_dir, volume_name)
+
+        found_db, dumped_any = _backup_dumps_for_volume(
+            containers=containers,
+            vol_dir=vol_dir,
+            databases_df=databases_df,
+            database_containers=args.database_containers,
+        )
+
+        # dump-only-sql logic:
+        if args.dump_only_sql:
+            if found_db:
+                if not dumped_any:
+                    print(
+                        f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. Falling back to file backup.",
+                        flush=True,
+                    )
+                    # fall through to file backup below
+                else:
+                    # DB volume successfully dumped -> skip file backup
+                    continue
+            # Non-DB volume -> always do file backup (fall through)
+
+        if args.everything:
+            # "everything": always do pre-rsync, then stop + rsync again
+            backup_volume(versions_dir, volume_name, vol_dir)
+            change_containers_status(containers, "stop")
+            backup_volume(versions_dir, volume_name, vol_dir)
+            if not args.shutdown:
+                change_containers_status(containers, "start")
+            continue
+
+        # default: rsync, and if needed stop + rsync
+        backup_volume(versions_dir, volume_name, vol_dir)
+        if requires_stop(containers, args.images_no_stop_required):
+            change_containers_status(containers, "stop")
+            backup_volume(versions_dir, volume_name, vol_dir)
+            if not args.shutdown:
+                change_containers_status(containers, "start")
+
+    # Stamp the backup version directory using dirval (python lib)
+    stamp_directory(version_dir)
+    print("Finished volume backups.", flush=True)
+
+    print("Handling Docker Compose services...", flush=True)
+    handle_docker_compose_services(
+        args.compose_dir, args.docker_compose_hard_restart_required
+    )
+
+    return 0

src/baudolo/backup/cli.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import argparse
+import os
+
+
+def parse_args() -> argparse.Namespace:
+    dirname = os.path.dirname(__file__)
+    default_databases_csv = os.path.join(dirname, "databases.csv")
+
+    p = argparse.ArgumentParser(description="Backup Docker volumes.")
+
+    p.add_argument(
+        "--compose-dir",
+        type=str,
+        required=True,
+        help="Path to the parent directory containing docker-compose setups",
+    )
+    p.add_argument(
+        "--docker-compose-hard-restart-required",
+        nargs="+",
+        default=["mailu"],
+        help="Compose dir names that require 'docker-compose down && up -d' (default: mailu)",
+    )
+
+    p.add_argument(
+        "--repo-name",
+        default="backup-docker-to-local",
+        help="Backup repo folder name under <backups-dir>/<machine-id>/ (default: git repo folder name)",
+    )
+    p.add_argument(
+        "--databases-csv",
+        default=default_databases_csv,
+        help=f"Path to databases.csv (default: {default_databases_csv})",
+    )
+    p.add_argument(
+        "--backups-dir",
+        default="/var/lib/backup/",
+        help="Backup root directory (default: /var/lib/backup/)",
+    )
+
+    p.add_argument(
+        "--database-containers",
+        nargs="+",
+        required=True,
+        help="Container names treated as special instances for database backups",
+    )
+    p.add_argument(
+        "--images-no-stop-required",
+        nargs="+",
+        required=True,
+        help="Image name patterns for which containers should not be stopped during file backup",
+    )
+    p.add_argument(
+        "--images-no-backup-required",
+        nargs="+",
+        default=[],
+        help="Image name patterns for which no backup should be performed",
+    )
+
+    p.add_argument(
+        "--everything",
+        action="store_true",
+        help="Force file backup for all volumes and also execute database dumps (like old script)",
+    )
+    p.add_argument(
+        "--shutdown",
+        action="store_true",
+        help="Do not restart containers after backup",
+    )
+    
+    p.add_argument(
+        "--dump-only-sql",
+        action="store_true",
+        help=(
+            "Create database dumps only for DB volumes. "
+            "File backups are skipped for DB volumes if a dump succeeds, "
+            "but non-DB volumes are still backed up. "
+            "If a DB dump cannot be produced, baudolo falls back to a file backup."
+        ),
+    )
+    return p.parse_args()

src/baudolo/backup/compose.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import os
+import subprocess
+
+
+def hard_restart_docker_services(dir_path: str) -> None:
+    print(f"Hard restart docker-compose services in: {dir_path}", flush=True)
+    subprocess.run(["docker-compose", "down"], cwd=dir_path, check=True)
+    subprocess.run(["docker-compose", "up", "-d"], cwd=dir_path, check=True)
+
+
+def handle_docker_compose_services(
+    parent_directory: str, hard_restart_required: list[str]
+) -> None:
+    for entry in os.scandir(parent_directory):
+        if not entry.is_dir():
+            continue
+
+        dir_path = entry.path
+        name = os.path.basename(dir_path)
+        compose_file = os.path.join(dir_path, "docker-compose.yml")
+
+        print(f"Checking directory: {dir_path}", flush=True)
+        if not os.path.isfile(compose_file):
+            print("No docker-compose.yml found. Skipping.", flush=True)
+            continue
+
+        if name in hard_restart_required:
+            print(f"{name}: hard restart required.", flush=True)
+            hard_restart_docker_services(dir_path)
+        else:
+            print(f"{name}: no restart required.", flush=True)

src/baudolo/backup/db.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import os
+import pathlib
+import re
+import logging
+from typing import Optional
+
+import pandas
+
+from .shell import BackupException, execute_shell_command
+
+log = logging.getLogger(__name__)
+
+
+def get_instance(container: str, database_containers: list[str]) -> str:
+    """
+    Derive a stable instance name from the container name.
+    """
+    if container in database_containers:
+        return container
+    return re.split(r"(_|-)(database|db|postgres)", container)[0]
+
+
+def _validate_database_value(value: Optional[str], *, instance: str) -> str:
+    """
+    Enforce explicit database semantics:
+
+    - "*"       => dump ALL databases (cluster dump for Postgres)
+    - "<name>"  => dump exactly this database
+    - ""        => invalid configuration (would previously result in NaN / nan.backup.sql)
+    """
+    v = (value or "").strip()
+    if v == "":
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': "
+            "column 'database' must be '*' or a concrete database name (not empty)."
+        )
+    return v
+
+
+def _atomic_write_cmd(cmd: str, out_file: str) -> None:
+    """
+    Write dump output atomically:
+    - write to <file>.tmp
+    - rename to <file> only on success
+
+    This prevents empty or partial dump files from being treated as valid backups.
+    """
+    tmp = f"{out_file}.tmp"
+    execute_shell_command(f"{cmd} > {tmp}")
+    execute_shell_command(f"mv {tmp} {out_file}")
+
+
+def fallback_pg_dumpall(container: str, username: str, password: str, out_file: str) -> None:
+    """
+    Perform a full Postgres cluster dump using pg_dumpall.
+    """
+    cmd = (
+        f"PGPASSWORD={password} docker exec -i {container} "
+        f"pg_dumpall -U {username} -h localhost"
+    )
+    _atomic_write_cmd(cmd, out_file)
+
+
+def backup_database(
+    *,
+    container: str,
+    volume_dir: str,
+    db_type: str,
+    databases_df: "pandas.DataFrame",
+    database_containers: list[str],
+) -> bool:
+    """
+    Backup databases for a given DB container.
+
+    Returns True if at least one dump was produced.
+    """
+    instance_name = get_instance(container, database_containers)
+
+    entries = databases_df[databases_df["instance"] == instance_name]
+    if entries.empty:
+        log.debug("No database entries for instance '%s'", instance_name)
+        return False
+
+    out_dir = os.path.join(volume_dir, "sql")
+    pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
+
+    produced = False
+
+    for row in entries.itertuples(index=False):
+        raw_db = getattr(row, "database", "")
+        user = (getattr(row, "username", "") or "").strip()
+        password = (getattr(row, "password", "") or "").strip()
+
+        db_value = _validate_database_value(raw_db, instance=instance_name)
+
+        # Explicit: dump ALL databases
+        if db_value == "*":
+            if db_type != "postgres":
+                raise ValueError(
+                    f"databases.csv entry for instance '{instance_name}': "
+                    "'*' is currently only supported for Postgres."
+                )
+
+            cluster_file = os.path.join(
+                out_dir, f"{instance_name}.cluster.backup.sql"
+            )
+            fallback_pg_dumpall(container, user, password, cluster_file)
+            produced = True
+            continue
+
+        # Concrete database dump
+        db_name = db_value
+        dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
+
+        if db_type == "mariadb":
+            cmd = (
+                f"docker exec {container} /usr/bin/mariadb-dump "
+                f"-u {user} -p{password} {db_name}"
+            )
+            _atomic_write_cmd(cmd, dump_file)
+            produced = True
+            continue
+
+        if db_type == "postgres":
+            try:
+                cmd = (
+                    f"PGPASSWORD={password} docker exec -i {container} "
+                    f"pg_dump -U {user} -d {db_name} -h localhost"
+                )
+                _atomic_write_cmd(cmd, dump_file)
+                produced = True
+            except BackupException as e:
+                # Explicit DB dump failed -> hard error
+                raise BackupException(
+                    f"Postgres dump failed for instance '{instance_name}', "
+                    f"database '{db_name}'. This database was explicitly configured "
+                    "and therefore must succeed.\n"
+                    f"{e}"
+                )
+            continue
+
+    return produced

src/baudolo/backup/docker.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from .shell import execute_shell_command
+
+
+def get_image_info(container: str) -> str:
+    return execute_shell_command(
+        f"docker inspect --format '{{{{.Config.Image}}}}' {container}"
+    )[0]
+
+
+def has_image(container: str, pattern: str) -> bool:
+    """Return True if container's image contains the pattern."""
+    return pattern in get_image_info(container)
+
+
+def docker_volume_names() -> list[str]:
+    return execute_shell_command("docker volume ls --format '{{.Name}}'")
+
+
+def containers_using_volume(volume_name: str) -> list[str]:
+    return execute_shell_command(
+        f"docker ps --filter volume=\"{volume_name}\" --format '{{{{.Names}}}}'"
+    )
+
+
+def change_containers_status(containers: list[str], status: str) -> None:
+    """Stop or start a list of containers."""
+    if not containers:
+        print(f"No containers to {status}.", flush=True)
+        return
+    names = " ".join(containers)
+    print(f"{status.capitalize()} containers: {names}...", flush=True)
+    execute_shell_command(f"docker {status} {names}")
+
+
+def docker_volume_exists(volume: str) -> bool:
+    # Avoid throwing exceptions for exists checks.
+    try:
+        execute_shell_command(
+            f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK"
+        )
+        return True
+    except Exception:
+        return False

src/baudolo/backup/shell.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import subprocess
+
+
+class BackupException(Exception):
+    """Generic exception for backup errors."""
+
+
+def execute_shell_command(command: str) -> list[str]:
+    """Execute a shell command and return its output lines."""
+    print(command, flush=True)
+    process = subprocess.Popen(
+        [command],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        shell=True,
+    )
+    out, err = process.communicate()
+    if process.returncode != 0:
+        raise BackupException(
+            f"Error in command: {command}\n"
+            f"Output: {out}\nError: {err}\n"
+            f"Exit code: {process.returncode}"
+        )
+    return [line.decode("utf-8") for line in out.splitlines()]

src/baudolo/backup/volume.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import os
+import pathlib
+
+from .shell import BackupException, execute_shell_command
+
+
+def get_storage_path(volume_name: str) -> str:
+    path = execute_shell_command(
+        f"docker volume inspect --format '{{{{ .Mountpoint }}}}' {volume_name}"
+    )[0]
+    return f"{path}/"
+
+
+def get_last_backup_dir(
+    versions_dir: str, volume_name: str, current_backup_dir: str
+) -> str | None:
+    versions = sorted(os.listdir(versions_dir), reverse=True)
+    for version in versions:
+        candidate = os.path.join(versions_dir, version, volume_name, "files", "")
+        if candidate != current_backup_dir and os.path.isdir(candidate):
+            return candidate
+    return None
+
+
+def backup_volume(versions_dir: str, volume_name: str, volume_dir: str) -> None:
+    """Perform incremental file backup of a Docker volume."""
+    dest = os.path.join(volume_dir, "files") + "/"
+    pathlib.Path(dest).mkdir(parents=True, exist_ok=True)
+
+    last = get_last_backup_dir(versions_dir, volume_name, dest)
+    link_dest = f"--link-dest='{last}'" if last else ""
+    source = get_storage_path(volume_name)
+
+    cmd = f"rsync -abP --delete --delete-excluded {link_dest} {source} {dest}"
+
+    try:
+        execute_shell_command(cmd)
+    except BackupException as e:
+        if "file has vanished" in str(e):
+            print(
+                "Warning: Some files vanished before transfer. Continuing.", flush=True
+            )
+        else:
+            raise

src/baudolo/restore/__init__.py

@@ -0,0 +1 @@
+__all__ = ["main"]

src/baudolo/restore/__main__.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import argparse
+import sys
+
+from .paths import BackupPaths
+from .files import restore_volume_files
+from .db.postgres import restore_postgres_sql
+from .db.mariadb import restore_mariadb_sql
+
+
+def _add_common_backup_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("volume_name", help="Docker volume name (target volume)")
+    p.add_argument("backup_hash", help="Hashed machine id")
+    p.add_argument("version", help="Backup version directory name")
+
+    p.add_argument(
+        "--backups-dir",
+        default="/Backups",
+        help="Backup root directory (default: /Backups)",
+    )
+    p.add_argument(
+        "--repo-name",
+        default="backup-docker-to-local",
+        help="Backup repo folder name under <backups-dir>/<hash>/ (default: backup-docker-to-local)",
+    )
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="baudolo-restore",
+        description="Restore docker volume files and DB dumps.",
+    )
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    # ------------------------------------------------------------------
+    # files
+    # ------------------------------------------------------------------
+    p_files = sub.add_parser("files", help="Restore files into a docker volume")
+    _add_common_backup_args(p_files)
+    p_files.add_argument(
+        "--rsync-image",
+        default="ghcr.io/kevinveenbirkenbach/alpine-rsync",
+    )
+    p_files.add_argument(
+        "--source-volume",
+        default=None,
+        help=(
+            "Volume name used as backup source path key. "
+            "Defaults to <volume_name> (target volume). "
+            "Use this when restoring from one volume backup into a different target volume."
+        ),
+    )
+
+    # ------------------------------------------------------------------
+    # postgres
+    # ------------------------------------------------------------------
+    p_pg = sub.add_parser("postgres", help="Restore a single PostgreSQL database dump")
+    _add_common_backup_args(p_pg)
+    p_pg.add_argument("--container", required=True)
+    p_pg.add_argument("--db-name", required=True)
+    p_pg.add_argument("--db-user", default=None, help="Defaults to db-name if omitted")
+    p_pg.add_argument("--db-password", required=True)
+    p_pg.add_argument("--empty", action="store_true")
+
+    # ------------------------------------------------------------------
+    # mariadb
+    # ------------------------------------------------------------------
+    p_mdb = sub.add_parser(
+        "mariadb", help="Restore a single MariaDB/MySQL-compatible dump"
+    )
+    _add_common_backup_args(p_mdb)
+    p_mdb.add_argument("--container", required=True)
+    p_mdb.add_argument("--db-name", required=True)
+    p_mdb.add_argument("--db-user", default=None, help="Defaults to db-name if omitted")
+    p_mdb.add_argument("--db-password", required=True)
+    p_mdb.add_argument("--empty", action="store_true")
+
+    args = parser.parse_args(argv)
+
+    try:
+        if args.cmd == "files":
+            # target volume = args.volume_name
+            # source volume (backup key) defaults to target volume
+            source_volume = args.source_volume or args.volume_name
+
+            bp_files = BackupPaths(
+                source_volume,
+                args.backup_hash,
+                args.version,
+                repo_name=args.repo_name,
+                backups_dir=args.backups_dir,
+            )
+
+            return restore_volume_files(
+                args.volume_name,
+                bp_files.files_dir(),
+                rsync_image=args.rsync_image,
+            )
+
+        if args.cmd == "postgres":
+            user = args.db_user or args.db_name
+            restore_postgres_sql(
+                container=args.container,
+                db_name=args.db_name,
+                user=user,
+                password=args.db_password,
+                sql_path=BackupPaths(
+                    args.volume_name,
+                    args.backup_hash,
+                    args.version,
+                    repo_name=args.repo_name,
+                    backups_dir=args.backups_dir,
+                ).sql_file(args.db_name),
+                empty=args.empty,
+            )
+            return 0
+
+        if args.cmd == "mariadb":
+            user = args.db_user or args.db_name
+            restore_mariadb_sql(
+                container=args.container,
+                db_name=args.db_name,
+                user=user,
+                password=args.db_password,
+                sql_path=BackupPaths(
+                    args.volume_name,
+                    args.backup_hash,
+                    args.version,
+                    repo_name=args.repo_name,
+                    backups_dir=args.backups_dir,
+                ).sql_file(args.db_name),
+                empty=args.empty,
+            )
+            return 0
+
+        parser.error("Unhandled command")
+        return 2
+
+    except Exception as e:
+        print(f"ERROR: {e}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/baudolo/restore/db/__init__.py

@@ -0,0 +1 @@
+"""Database restore handlers (Postgres, MariaDB/MySQL)."""

src/baudolo/restore/db/mariadb.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import os
+import sys
+
+from ..run import docker_exec, docker_exec_sh
+
+
+def _pick_client(container: str) -> str:
+    """
+    Prefer 'mariadb', fallback to 'mysql'.
+    Some MariaDB images no longer ship a 'mysql' binary, so we must not assume it exists.
+    """
+    script = r"""
+set -eu
+if command -v mariadb >/dev/null 2>&1; then echo mariadb; exit 0; fi
+if command -v mysql   >/dev/null 2>&1; then echo mysql;   exit 0; fi
+exit 42
+"""
+    try:
+        out = docker_exec_sh(container, script, capture=True).stdout.decode().strip()
+        if not out:
+            raise RuntimeError("empty client detection output")
+        return out
+    except Exception as e:
+        print(
+            "ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr
+        )
+        raise e
+
+
+def restore_mariadb_sql(
+    *,
+    container: str,
+    db_name: str,
+    user: str,
+    password: str,
+    sql_path: str,
+    empty: bool,
+) -> None:
+    client = _pick_client(container)
+
+    if not os.path.isfile(sql_path):
+        raise FileNotFoundError(sql_path)
+
+    if empty:
+        # IMPORTANT:
+        # Do NOT hardcode 'mysql' here. Use the detected client.
+        # MariaDB 11 images may not contain the mysql binary at all.
+        docker_exec(
+            container,
+            [
+                client,
+                "-u",
+                user,
+                f"--password={password}",
+                "-e",
+                "SET FOREIGN_KEY_CHECKS=0;",
+            ],
+        )
+
+        result = docker_exec(
+            container,
+            [
+                client,
+                "-u",
+                user,
+                f"--password={password}",
+                "-N",
+                "-e",
+                f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{db_name}';",
+            ],
+            capture=True,
+        )
+        tables = result.stdout.decode().split()
+
+        for tbl in tables:
+            docker_exec(
+                container,
+                [
+                    client,
+                    "-u",
+                    user,
+                    f"--password={password}",
+                    "-e",
+                    f"DROP TABLE IF EXISTS `{db_name}`.`{tbl}`;",
+                ],
+            )
+
+        docker_exec(
+            container,
+            [
+                client,
+                "-u",
+                user,
+                f"--password={password}",
+                "-e",
+                "SET FOREIGN_KEY_CHECKS=1;",
+            ],
+        )
+
+    with open(sql_path, "rb") as f:
+        docker_exec(
+            container, [client, "-u", user, f"--password={password}", db_name], stdin=f
+        )
+
+    print(f"MariaDB/MySQL restore complete for db '{db_name}'.")

src/baudolo/restore/db/postgres.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import os
+
+from ..run import docker_exec
+
+
+def restore_postgres_sql(
+    *,
+    container: str,
+    db_name: str,
+    user: str,
+    password: str,
+    sql_path: str,
+    empty: bool,
+) -> None:
+    if not os.path.isfile(sql_path):
+        raise FileNotFoundError(sql_path)
+
+    # Make password available INSIDE the container for psql.
+    docker_env = {"PGPASSWORD": password}
+
+    if empty:
+        drop_sql = r"""
+DO $$ DECLARE r RECORD;
+BEGIN
+  FOR r IN (
+    SELECT table_name AS name, 'TABLE' AS type FROM information_schema.tables WHERE table_schema='public'
+    UNION ALL
+    SELECT routine_name AS name, 'FUNCTION' AS type FROM information_schema.routines WHERE specific_schema='public'
+    UNION ALL
+    SELECT sequence_name AS name, 'SEQUENCE' AS type FROM information_schema.sequences WHERE sequence_schema='public'
+  ) LOOP
+    EXECUTE format('DROP %s public.%I CASCADE', r.type, r.name);
+  END LOOP;
+END $$;
+"""
+        docker_exec(
+            container,
+            ["psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", db_name],
+            stdin=drop_sql.encode(),
+            docker_env=docker_env,
+        )
+
+    with open(sql_path, "rb") as f:
+        docker_exec(
+            container,
+            ["psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", db_name],
+            stdin=f,
+            docker_env=docker_env,
+        )
+
+    print(f"PostgreSQL restore complete for db '{db_name}'.")

src/baudolo/restore/files.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import os
+import sys
+
+from .run import run, docker_volume_exists
+
+
+def restore_volume_files(
+    volume_name: str, backup_files_dir: str, *, rsync_image: str
+) -> int:
+    if not os.path.isdir(backup_files_dir):
+        print(f"ERROR: backup files dir not found: {backup_files_dir}", file=sys.stderr)
+        return 2
+
+    if not docker_volume_exists(volume_name):
+        print(f"Volume {volume_name} does not exist. Creating...")
+        run(["docker", "volume", "create", volume_name])
+    else:
+        print(f"Volume {volume_name} already exists.")
+
+    # Keep behavior close to the old script: rsync -avv --delete
+    run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            "-v",
+            f"{volume_name}:/recover/",
+            "-v",
+            f"{backup_files_dir}:/backup/",
+            rsync_image,
+            "sh",
+            "-lc",
+            "rsync -avv --delete /backup/ /recover/",
+        ]
+    )
+    print("File restore complete.")
+    return 0

src/baudolo/restore/paths.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class BackupPaths:
+    volume_name: str
+    backup_hash: str
+    version: str
+    repo_name: str
+    backups_dir: str = "/Backups"
+
+    def root(self) -> str:
+        # Always build an absolute path under backups_dir
+        return os.path.join(
+            self.backups_dir,
+            self.backup_hash,
+            self.repo_name,
+            self.version,
+            self.volume_name,
+        )
+
+    def files_dir(self) -> str:
+        return os.path.join(self.root(), "files")
+
+    def sql_file(self, db_name: str) -> str:
+        return os.path.join(self.root(), "sql", f"{db_name}.backup.sql")

src/baudolo/restore/run.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+from typing import Optional
+
+
+def run(
+    cmd: list[str],
+    *,
+    stdin=None,
+    capture: bool = False,
+    env: Optional[dict] = None,
+) -> subprocess.CompletedProcess:
+    try:
+        kwargs: dict = {
+            "check": True,
+            "capture_output": capture,
+            "env": env,
+        }
+
+        # If stdin is raw data (bytes/str), pass it via input=.
+        # IMPORTANT: when using input=..., do NOT pass stdin=... as well.
+        if isinstance(stdin, (bytes, str)):
+            kwargs["input"] = stdin
+        else:
+            kwargs["stdin"] = stdin
+
+        return subprocess.run(cmd, **kwargs)
+
+    except subprocess.CalledProcessError as e:
+        msg = f"ERROR: command failed ({e.returncode}): {' '.join(cmd)}"
+        print(msg, file=sys.stderr)
+        if e.stdout:
+            try:
+                print(e.stdout.decode(), file=sys.stderr)
+            except Exception:
+                print(e.stdout, file=sys.stderr)
+        if e.stderr:
+            try:
+                print(e.stderr.decode(), file=sys.stderr)
+            except Exception:
+                print(e.stderr, file=sys.stderr)
+        raise
+
+
+def docker_exec(
+    container: str,
+    argv: list[str],
+    *,
+    stdin=None,
+    capture: bool = False,
+    env: Optional[dict] = None,
+    docker_env: Optional[dict[str, str]] = None,
+) -> subprocess.CompletedProcess:
+    cmd: list[str] = ["docker", "exec", "-i"]
+    if docker_env:
+        for k, v in docker_env.items():
+            cmd.extend(["-e", f"{k}={v}"])
+    cmd.extend([container, *argv])
+    return run(cmd, stdin=stdin, capture=capture, env=env)
+
+
+def docker_exec_sh(
+    container: str,
+    script: str,
+    *,
+    stdin=None,
+    capture: bool = False,
+    env: Optional[dict] = None,
+    docker_env: Optional[dict[str, str]] = None,
+) -> subprocess.CompletedProcess:
+    return docker_exec(
+        container,
+        ["sh", "-lc", script],
+        stdin=stdin,
+        capture=capture,
+        env=env,
+        docker_env=docker_env,
+    )
+
+
+def docker_volume_exists(volume: str) -> bool:
+    p = subprocess.run(
+        ["docker", "volume", "inspect", volume],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    return p.returncode == 0

src/baudolo/seed/__main__.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+import pandas as pd
+from typing import Optional
+
+
+DB_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
+
+def _validate_database_value(value: Optional[str], *, instance: str) -> str:
+    v = (value or "").strip()
+    if v == "":
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': "
+            "column 'database' must be '*' or a concrete database name (not empty)."
+        )
+    if v == "*":
+        return "*"
+    if v.lower() == "nan":
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': database must not be 'nan'."
+        )
+    if not DB_NAME_RE.match(v):
+        raise ValueError(
+            f"Invalid databases.csv entry for instance '{instance}': "
+            f"invalid database name '{v}'. Allowed: letters, numbers, '_' and '-'."
+        )
+    return v
+
+def check_and_add_entry(
+    file_path: str,
+    instance: str,
+    database: Optional[str],
+    username: str,
+    password: str,
+) -> None:
+    """
+    Add or update an entry in databases.csv.
+
+    The function enforces strict validation:
+    - database MUST be set
+    - database MUST be '*' or a valid database name
+    """
+    database = _validate_database_value(database, instance=instance)
+
+    if os.path.exists(file_path):
+        df = pd.read_csv(
+            file_path,
+            sep=";",
+            dtype=str,
+            keep_default_na=False,
+        )
+    else:
+        df = pd.DataFrame(
+            columns=["instance", "database", "username", "password"]
+        )
+
+    mask = (df["instance"] == instance) & (df["database"] == database)
+
+    if mask.any():
+        print("Updating existing entry.")
+        df.loc[mask, ["username", "password"]] = [username, password]
+    else:
+        print("Adding new entry.")
+        new_entry = pd.DataFrame(
+            [[instance, database, username, password]],
+            columns=["instance", "database", "username", "password"],
+        )
+        df = pd.concat([df, new_entry], ignore_index=True)
+
+    df.to_csv(file_path, sep=";", index=False)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Seed or update databases.csv for backup configuration."
+    )
+    parser.add_argument("file", help="Path to databases.csv")
+    parser.add_argument("instance", help="Instance name (e.g. bigbluebutton)")
+    parser.add_argument(
+        "database",
+        help="Database name or '*' to dump all databases",
+    )
+    parser.add_argument("username", help="Database username")
+    parser.add_argument("password", help="Database password")
+
+    args = parser.parse_args()
+
+    try:
+        check_and_add_entry(
+            file_path=args.file,
+            instance=args.instance,
+            database=args.database,
+            username=args.username,
+            password=args.password,
+        )
+    except Exception as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tests/e2e/helpers.py

@@ -0,0 +1,257 @@
+# tests/e2e/helpers.py
+from __future__ import annotations
+
+import shutil
+import subprocess
+import time
+import uuid
+from pathlib import Path
+
+
+def run(
+    cmd: list[str],
+    *,
+    capture: bool = True,
+    check: bool = True,
+    cwd: str | None = None,
+) -> subprocess.CompletedProcess:
+    try:
+        return subprocess.run(
+            cmd,
+            check=check,
+            cwd=cwd,
+            text=True,
+            capture_output=capture,
+        )
+    except subprocess.CalledProcessError as e:
+        # Print captured output so failing E2E tests are "live" / debuggable in CI logs
+        print(">>> command failed:", " ".join(cmd))
+        print(">>> exit code:", e.returncode)
+        if e.stdout:
+            print(">>> STDOUT:\n" + e.stdout)
+        if e.stderr:
+            print(">>> STDERR:\n" + e.stderr)
+        raise
+
+
+def sh(
+    cmd: str, *, capture: bool = True, check: bool = True
+) -> subprocess.CompletedProcess:
+    return run(["sh", "-lc", cmd], capture=capture, check=check)
+
+
+def unique(prefix: str) -> str:
+    return f"{prefix}-{uuid.uuid4().hex[:10]}"
+
+
+def require_docker() -> None:
+    run(["docker", "version"], capture=True, check=True)
+
+
+def machine_hash() -> str:
+    out = sh("sha256sum /etc/machine-id | awk '{print $1}'").stdout.strip()
+    if len(out) < 16:
+        raise RuntimeError("Could not determine machine hash from /etc/machine-id")
+    return out
+
+
+def wait_for_log(container: str, pattern: str, timeout_s: int = 60) -> None:
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        p = run(["docker", "logs", container], capture=True, check=False)
+        if pattern in (p.stdout or ""):
+            return
+        time.sleep(1)
+    raise TimeoutError(f"Timed out waiting for log pattern '{pattern}' in {container}")
+
+
+def wait_for_postgres(
+    container: str, *, user: str = "postgres", timeout_s: int = 90
+) -> None:
+    """
+    Docker-outside-of-Docker friendly readiness: check from inside the DB container.
+    """
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        p = run(
+            [
+                "docker",
+                "exec",
+                container,
+                "sh",
+                "-lc",
+                f"pg_isready -U {user} -h localhost",
+            ],
+            capture=True,
+            check=False,
+        )
+        if p.returncode == 0:
+            return
+        time.sleep(1)
+    raise TimeoutError(
+        f"Timed out waiting for Postgres readiness in container {container}"
+    )
+
+
+def wait_for_mariadb(
+    container: str, *, root_password: str, timeout_s: int = 90
+) -> None:
+    """
+    Liveness probe for MariaDB.
+
+    IMPORTANT (MariaDB 11):
+    Root TCP auth is often restricted (unix_socket auth), so a TCP ping like
+    `mariadb-admin -uroot -p... -h localhost ping` can fail even though the server is up.
+    We therefore check readiness via a socket-based query.
+    """
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        p = run(
+            [
+                "docker",
+                "exec",
+                container,
+                "sh",
+                "-lc",
+                'mariadb -uroot --protocol=socket -e "SELECT 1;"',
+            ],
+            capture=True,
+            check=False,
+        )
+        if p.returncode == 0:
+            return
+        time.sleep(1)
+    raise TimeoutError(
+        f"Timed out waiting for MariaDB readiness in container {container}"
+    )
+
+
+def wait_for_mariadb_sql(
+    container: str, *, user: str, password: str, timeout_s: int = 90
+) -> None:
+    """
+    SQL login readiness for the *dedicated test user* over TCP.
+
+    This is separate from wait_for_mariadb(root) because root may be socket-only,
+    while the tests use a normal user that should work via TCP.
+    """
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        p = run(
+            [
+                "docker",
+                "exec",
+                container,
+                "sh",
+                "-lc",
+                f'mariadb -h 127.0.0.1 -u{user} -p{password} -e "SELECT 1;"',
+            ],
+            capture=True,
+            check=False,
+        )
+        if p.returncode == 0:
+            return
+        time.sleep(1)
+    raise TimeoutError(
+        f"Timed out waiting for MariaDB SQL login readiness in container {container}"
+    )
+
+
+def backup_run(
+    *,
+    backups_dir: str,
+    repo_name: str,
+    compose_dir: str,
+    databases_csv: str,
+    database_containers: list[str],
+    images_no_stop_required: list[str],
+    images_no_backup_required: list[str] | None = None,
+    dump_only_sql: bool = False,
+) -> None:
+    cmd = [
+        "baudolo",
+        "--compose-dir",
+        compose_dir,
+        "--docker-compose-hard-restart-required",
+        "mailu",
+        "--repo-name",
+        repo_name,
+        "--databases-csv",
+        databases_csv,
+        "--backups-dir",
+        backups_dir,
+        "--database-containers",
+        *database_containers,
+        "--images-no-stop-required",
+        *images_no_stop_required,
+    ]
+    if images_no_backup_required:
+        cmd += ["--images-no-backup-required", *images_no_backup_required]
+    if dump_only_sql:
+        cmd += ["--dump-only-sql"]
+
+    try:
+        run(cmd, capture=True, check=True)
+    except subprocess.CalledProcessError as e:
+        print(">>> baudolo failed (exit code:", e.returncode, ")")
+        if e.stdout:
+            print(">>> baudolo STDOUT:\n" + e.stdout)
+        if e.stderr:
+            print(">>> baudolo STDERR:\n" + e.stderr)
+        raise
+
+
+def latest_version_dir(backups_dir: str, repo_name: str) -> tuple[str, str]:
+    """
+    Returns (hash, version) for the latest backup.
+    """
+    h = machine_hash()
+    root = Path(backups_dir) / h / repo_name
+    if not root.is_dir():
+        raise FileNotFoundError(str(root))
+
+    versions = sorted([p.name for p in root.iterdir() if p.is_dir()])
+    if not versions:
+        raise RuntimeError(f"No versions found under {root}")
+    return h, versions[-1]
+
+
+def backup_path(backups_dir: str, repo_name: str, version: str, volume: str) -> Path:
+    h = machine_hash()
+    return Path(backups_dir) / h / repo_name / version / volume
+
+
+def create_minimal_compose_dir(base: str) -> str:
+    """
+    baudolo requires --compose-dir. Create an empty dir with one non-compose subdir.
+    """
+    p = Path(base) / "compose-root"
+    p.mkdir(parents=True, exist_ok=True)
+    (p / "noop").mkdir(parents=True, exist_ok=True)
+    return str(p)
+
+
+def write_databases_csv(path: str, rows: list[tuple[str, str, str, str]]) -> None:
+    """
+    rows: (instance, database, username, password)
+    database may be '' (empty) to trigger pg_dumpall behavior if you want, but here we use db name.
+    """
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        f.write("instance;database;username;password\n")
+        for inst, db, user, pw in rows:
+            f.write(f"{inst};{db};{user};{pw}\n")
+
+
+def cleanup_docker(*, containers: list[str], volumes: list[str]) -> None:
+    for c in containers:
+        run(["docker", "rm", "-f", c], capture=True, check=False)
+    for v in volumes:
+        run(["docker", "volume", "rm", "-f", v], capture=True, check=False)
+
+
+def ensure_empty_dir(path: str) -> None:
+    p = Path(path)
+    if p.exists():
+        shutil.rmtree(p)
+    p.mkdir(parents=True, exist_ok=True)

tests/e2e/test_e2e_cli_contract_dump_only_sql.py

@@ -0,0 +1,29 @@
+import unittest
+
+from .helpers import run
+
+
+class TestE2ECLIContractDumpOnlySql(unittest.TestCase):
+    def test_help_mentions_new_flag(self) -> None:
+        cp = run(["baudolo", "--help"], capture=True, check=True)
+        out = (cp.stdout or "") + "\n" + (cp.stderr or "")
+        self.assertIn(
+            "--dump-only-sql",
+            out,
+            f"Expected '--dump-only-sql' to appear in --help output. Output:\n{out}",
+        )
+
+    def test_old_flag_is_rejected(self) -> None:
+        cp = run(["baudolo", "--dump-only"], capture=True, check=False)
+        self.assertEqual(
+            cp.returncode,
+            2,
+            f"Expected exitcode 2 for unknown args, got {cp.returncode}\n"
+            f"STDOUT={cp.stdout}\nSTDERR={cp.stderr}",
+        )
+        err = (cp.stderr or "") + "\n" + (cp.stdout or "")
+        # Argparse typically prints "unrecognized arguments"
+        self.assertTrue(
+            ("unrecognized arguments" in err) or ("usage:" in err.lower()),
+            f"Expected argparse-style error output. Output:\n{err}",
+        )

tests/e2e/test_e2e_dump_only_fallback_to_files.py

@@ -0,0 +1,175 @@
+# tests/e2e/test_e2e_dump_only_fallback_to_files.py
+import unittest
+
+from .helpers import (
+    backup_path,
+    cleanup_docker,
+    create_minimal_compose_dir,
+    ensure_empty_dir,
+    latest_version_dir,
+    require_docker,
+    run,
+    unique,
+    write_databases_csv,
+    wait_for_postgres,
+)
+
+
+class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+        cls.prefix = unique("baudolo-e2e-dump-only-sql-fallback")
+        cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
+        ensure_empty_dir(cls.backups_dir)
+
+        cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
+        cls.repo_name = cls.prefix
+
+        cls.pg_container = f"{cls.prefix}-pg"
+        cls.pg_volume = f"{cls.prefix}-pg-vol"
+        cls.restore_volume = f"{cls.prefix}-restore-vol"
+
+        cls.containers = [cls.pg_container]
+        cls.volumes = [cls.pg_volume, cls.restore_volume]
+
+        run(["docker", "volume", "create", cls.pg_volume])
+
+        # Start Postgres (creates a real DB volume)
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.pg_container,
+                "-e",
+                "POSTGRES_PASSWORD=pgpw",
+                "-e",
+                "POSTGRES_DB=appdb",
+                "-e",
+                "POSTGRES_USER=postgres",
+                "-v",
+                f"{cls.pg_volume}:/var/lib/postgresql/data",
+                "postgres:16",
+            ]
+        )
+        wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
+
+        # Add a deterministic marker file into the volume
+        cls.marker = "dump-only-sql-fallback-marker"
+        run(
+            [
+                "docker",
+                "exec",
+                cls.pg_container,
+                "sh",
+                "-lc",
+                f"echo '{cls.marker}' > /var/lib/postgresql/data/marker.txt",
+            ]
+        )
+
+        # databases.csv WITHOUT matching entry for this instance -> should skip dump
+        cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
+        write_databases_csv(cls.databases_csv, [])  # empty except header
+
+        # Run baudolo with --dump-only-sql and a DB container present:
+        # Expected: WARNING + FALLBACK to file backup (files/ must exist)
+        cmd = [
+            "baudolo",
+            "--compose-dir",
+            cls.compose_dir,
+            "--docker-compose-hard-restart-required",
+            "mailu",
+            "--repo-name",
+            cls.repo_name,
+            "--databases-csv",
+            cls.databases_csv,
+            "--backups-dir",
+            cls.backups_dir,
+            "--database-containers",
+            cls.pg_container,
+            "--images-no-stop-required",
+            "postgres",
+            "mariadb",
+            "mysql",
+            "alpine",
+            "--dump-only-sql",
+        ]
+        cp = run(cmd, capture=True, check=True)
+
+        cls.stdout = cp.stdout or ""
+        cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
+
+        # Restore files into a fresh volume to prove file backup happened
+        run(["docker", "volume", "create", cls.restore_volume])
+        run(
+            [
+                "baudolo-restore",
+                "files",
+                cls.restore_volume,
+                cls.hash,
+                cls.version,
+                "--backups-dir",
+                cls.backups_dir,
+                "--repo-name",
+                cls.repo_name,
+                "--source-volume",
+                cls.pg_volume,
+                "--rsync-image",
+                "ghcr.io/kevinveenbirkenbach/alpine-rsync",
+            ]
+        )
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_warns_about_missing_dump_in_dump_only_mode(self) -> None:
+        self.assertIn(
+            "WARNING: dump-only-sql requested but no DB dump was produced",
+            self.stdout,
+            f"Expected warning in baudolo output. STDOUT:\n{self.stdout}",
+        )
+
+    def test_files_backup_exists_due_to_fallback(self) -> None:
+        p = backup_path(
+            self.backups_dir,
+            self.repo_name,
+            self.version,
+            self.pg_volume,
+        ) / "files"
+        self.assertTrue(p.is_dir(), f"Expected files backup dir at: {p}")
+
+    def test_sql_dump_not_present(self) -> None:
+        # There should be no sql dumps because databases.csv had no matching entry.
+        sql_dir = backup_path(
+            self.backups_dir,
+            self.repo_name,
+            self.version,
+            self.pg_volume,
+        ) / "sql"
+        # Could exist (dir created) in some edge cases, but should contain no *.sql dumps.
+        if sql_dir.exists():
+            dumps = list(sql_dir.glob("*.sql"))
+            self.assertEqual(
+                len(dumps),
+                0,
+                f"Did not expect SQL dump files, found: {dumps}",
+            )
+
+    def test_restored_files_contain_marker(self) -> None:
+        p = run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "-v",
+                f"{self.restore_volume}:/data",
+                "alpine:3.20",
+                "sh",
+                "-lc",
+                "cat /data/marker.txt",
+            ]
+        )
+        self.assertEqual((p.stdout or "").strip(), self.marker)

tests/e2e/test_e2e_dump_only_sql_mixed_run.py

@@ -0,0 +1,182 @@
+import unittest
+
+from .helpers import (
+    backup_path,
+    cleanup_docker,
+    create_minimal_compose_dir,
+    ensure_empty_dir,
+    latest_version_dir,
+    require_docker,
+    run,
+    unique,
+    wait_for_postgres,
+    write_databases_csv,
+)
+
+
+class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+        cls.prefix = unique("baudolo-e2e-dump-only-sql-mixed-run")
+        cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
+        ensure_empty_dir(cls.backups_dir)
+
+        cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
+        cls.repo_name = cls.prefix
+
+        # --- Volumes ---
+        cls.db_volume = f"{cls.prefix}-vol-db"
+        cls.files_volume = f"{cls.prefix}-vol-files"
+
+        # Track for cleanup
+        cls.containers: list[str] = []
+        cls.volumes = [cls.db_volume, cls.files_volume]
+
+        # Create volumes
+        run(["docker", "volume", "create", cls.db_volume])
+        run(["docker", "volume", "create", cls.files_volume])
+
+        # Put a marker into the non-db volume
+        run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "-v",
+                f"{cls.files_volume}:/data",
+                "alpine:3.20",
+                "sh",
+                "-lc",
+                "echo 'hello-non-db' > /data/hello.txt",
+            ]
+        )
+
+        # --- Start Postgres container using the DB volume ---
+        cls.pg_container = f"{cls.prefix}-pg"
+        cls.containers.append(cls.pg_container)
+
+        cls.pg_password = "postgres"
+        cls.pg_db = "testdb"
+        cls.pg_user = "postgres"
+
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.pg_container,
+                "-e",
+                f"POSTGRES_PASSWORD={cls.pg_password}",
+                "-v",
+                f"{cls.db_volume}:/var/lib/postgresql/data",
+                "postgres:16-alpine",
+            ]
+        )
+        wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
+
+        # Create deterministic content in DB so dump is non-empty
+        run(
+            [
+                "docker",
+                "exec",
+                cls.pg_container,
+                "sh",
+                "-lc",
+                f'psql -U postgres -c "CREATE DATABASE {cls.pg_db};" || true',
+            ],
+            check=True,
+        )
+        run(
+            [
+                "docker",
+                "exec",
+                cls.pg_container,
+                "sh",
+                "-lc",
+                (
+                    f'psql -U postgres -d {cls.pg_db} -c '
+                    '"CREATE TABLE IF NOT EXISTS t (id INT PRIMARY KEY, v TEXT);'
+                    "INSERT INTO t(id,v) VALUES (1,'hello-db') "
+                    "ON CONFLICT (id) DO UPDATE SET v=EXCLUDED.v;\""
+                ),
+            ],
+            check=True,
+        )
+
+        # databases.csv with an entry => dump should succeed
+        cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
+        write_databases_csv(
+            cls.databases_csv,
+            [(cls.pg_container, cls.pg_db, cls.pg_user, cls.pg_password)],
+        )
+
+        # Run baudolo with dump-only-sql
+        cmd = [
+            "baudolo",
+            "--compose-dir",
+            cls.compose_dir,
+            "--databases-csv",
+            cls.databases_csv,
+            "--database-containers",
+            cls.pg_container,
+            "--images-no-stop-required",
+            "alpine",
+            "postgres",
+            "mariadb",
+            "mysql",
+            "--dump-only-sql",
+            "--backups-dir",
+            cls.backups_dir,
+            "--repo-name",
+            cls.repo_name,
+        ]
+        cp = run(cmd, capture=True, check=True)
+        cls.stdout = cp.stdout
+        cls.stderr = cp.stderr
+
+        cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_db_volume_has_dump_and_no_files_dir(self) -> None:
+        base = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume)
+
+        dumps = base / "sql"
+        files = base / "files"
+
+        self.assertTrue(dumps.exists(), f"Expected dumps dir for DB volume at: {dumps}")
+        self.assertFalse(
+            files.exists(),
+            f"Did not expect files dir for DB volume when dump succeeded at: {files}",
+        )
+
+        # Optional: at least one dump file exists
+        dump_files = list(dumps.glob("*.sql")) + list(dumps.glob("*.sql.gz"))
+        self.assertTrue(
+            dump_files,
+            f"Expected at least one SQL dump file in {dumps}, found none.",
+        )
+
+    def test_non_db_volume_has_files_dir(self) -> None:
+        base = backup_path(
+            self.backups_dir, self.repo_name, self.version, self.files_volume
+        )
+        files = base / "files"
+        self.assertTrue(
+            files.exists(),
+            f"Expected files dir for non-DB volume at: {files}",
+        )
+
+    def test_dump_only_sql_does_not_disable_non_db_files_backup(self) -> None:
+        # Regression guard: even with --dump-only-sql, non-DB volumes must still be backed up as files
+        base = backup_path(
+            self.backups_dir, self.repo_name, self.version, self.files_volume
+        )
+        self.assertTrue(
+            (base / "files").exists(),
+            f"Expected non-DB volume files backup to exist at: {base / 'files'}",
+        )

tests/e2e/test_e2e_files_full.py

@@ -0,0 +1,116 @@
+import unittest
+
+from .helpers import (
+    backup_run,
+    backup_path,
+    cleanup_docker,
+    create_minimal_compose_dir,
+    ensure_empty_dir,
+    latest_version_dir,
+    require_docker,
+    unique,
+    write_databases_csv,
+    run,
+)
+
+
+class TestE2EFilesFull(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+        cls.prefix = unique("baudolo-e2e-files-full")
+        cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
+        ensure_empty_dir(cls.backups_dir)
+
+        cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
+        cls.repo_name = cls.prefix
+
+        cls.volume_src = f"{cls.prefix}-vol-src"
+        cls.volume_dst = f"{cls.prefix}-vol-dst"
+        cls.containers = []
+        cls.volumes = [cls.volume_src, cls.volume_dst]
+
+        # create source volume with a file
+        run(["docker", "volume", "create", cls.volume_src])
+        run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "-v",
+                f"{cls.volume_src}:/data",
+                "alpine:3.20",
+                "sh",
+                "-lc",
+                "mkdir -p /data && echo 'hello' > /data/hello.txt",
+            ]
+        )
+
+        # databases.csv (unused, but required by CLI)
+        cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
+        write_databases_csv(cls.databases_csv, [])
+
+        # Run backup (files should be copied)
+        backup_run(
+            backups_dir=cls.backups_dir,
+            repo_name=cls.repo_name,
+            compose_dir=cls.compose_dir,
+            databases_csv=cls.databases_csv,
+            database_containers=["dummy-db"],
+            images_no_stop_required=["alpine", "postgres", "mariadb", "mysql"],
+        )
+
+        cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_files_backup_exists(self) -> None:
+        p = (
+            backup_path(
+                self.backups_dir,
+                self.repo_name,
+                self.version,
+                self.volume_src,
+            )
+            / "files"
+            / "hello.txt"
+        )
+        self.assertTrue(p.is_file(), f"Expected backed up file at: {p}")
+
+    def test_restore_files_into_new_volume(self) -> None:
+        # restore files from volume_src backup into volume_dst
+        run(
+            [
+                "baudolo-restore",
+                "files",
+                self.volume_dst,
+                self.hash,
+                self.version,
+                "--backups-dir",
+                self.backups_dir,
+                "--repo-name",
+                self.repo_name,
+                "--source-volume",
+                self.volume_src,
+                "--rsync-image",
+                "ghcr.io/kevinveenbirkenbach/alpine-rsync",
+            ]
+        )
+
+        # verify restored file exists in dst volume
+        p = run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "-v",
+                f"{self.volume_dst}:/data",
+                "alpine:3.20",
+                "sh",
+                "-lc",
+                "cat /data/hello.txt",
+            ]
+        )
+        self.assertEqual((p.stdout or "").strip(), "hello")

tests/e2e/test_e2e_files_no_copy.py

@@ -0,0 +1,119 @@
+import unittest
+
+from .helpers import (
+    backup_run,
+    backup_path,
+    cleanup_docker,
+    create_minimal_compose_dir,
+    ensure_empty_dir,
+    latest_version_dir,
+    require_docker,
+    unique,
+    write_databases_csv,
+    run,
+)
+
+
+class TestE2EFilesNoCopy(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+        cls.prefix = unique("baudolo-e2e-files-nocopy")
+        cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
+        ensure_empty_dir(cls.backups_dir)
+
+        cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
+        cls.repo_name = cls.prefix
+
+        cls.volume_src = f"{cls.prefix}-vol-src"
+        cls.containers: list[str] = []
+        cls.volumes = [cls.volume_src]
+
+        # Create source volume and write a marker file
+        run(["docker", "volume", "create", cls.volume_src])
+        run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "-v",
+                f"{cls.volume_src}:/data",
+                "alpine:3.20",
+                "sh",
+                "-lc",
+                "echo 'hello' > /data/hello.txt",
+            ]
+        )
+
+        cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
+        write_databases_csv(cls.databases_csv, [])
+
+        # dump-only-sql => non-DB volumes are STILL backed up as files
+        backup_run(
+            backups_dir=cls.backups_dir,
+            repo_name=cls.repo_name,
+            compose_dir=cls.compose_dir,
+            databases_csv=cls.databases_csv,
+            database_containers=["dummy-db"],
+            images_no_stop_required=["alpine", "postgres", "mariadb", "mysql"],
+            dump_only_sql=True,
+        )
+
+        cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
+
+        # Wipe the volume to ensure restore actually restores something
+        run(["docker", "volume", "rm", "-f", cls.volume_src])
+        run(["docker", "volume", "create", cls.volume_src])
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_files_backup_present_for_non_db_volume(self) -> None:
+        p = (
+            backup_path(self.backups_dir, self.repo_name, self.version, self.volume_src)
+            / "files"
+        )
+        self.assertTrue(p.exists(), f"Expected files backup dir at: {p}")
+
+    def test_restore_files_succeeds_and_restores_content(self) -> None:
+        p = run(
+            [
+                "baudolo-restore",
+                "files",
+                self.volume_src,
+                self.hash,
+                self.version,
+                "--backups-dir",
+                self.backups_dir,
+                "--repo-name",
+                self.repo_name,
+            ],
+            check=False,
+        )
+        self.assertEqual(
+            p.returncode,
+            0,
+            f"Expected exitcode 0, got {p.returncode}\nSTDOUT={p.stdout}\nSTDERR={p.stderr}",
+        )
+
+        cp = run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "-v",
+                f"{self.volume_src}:/data",
+                "alpine:3.20",
+                "sh",
+                "-lc",
+                "cat /data/hello.txt",
+            ],
+            capture=True,
+            check=True,
+        )
+        self.assertEqual(
+            cp.stdout.strip(),
+            "hello",
+            f"Unexpected restored content. STDOUT={cp.stdout}\nSTDERR={cp.stderr}",
+        )

tests/e2e/test_e2e_images_no_backup_required_early_skip.py

@@ -0,0 +1,131 @@
+# tests/e2e/test_e2e_images_no_backup_required_early_skip.py
+import unittest
+
+from .helpers import (
+    backup_path,
+    cleanup_docker,
+    create_minimal_compose_dir,
+    ensure_empty_dir,
+    latest_version_dir,
+    require_docker,
+    run,
+    unique,
+    write_databases_csv,
+)
+
+
+class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+
+        cls.prefix = unique("baudolo-e2e-early-skip-no-backup-required")
+        cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
+        ensure_empty_dir(cls.backups_dir)
+
+        cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
+        cls.repo_name = cls.prefix
+
+        # --- Docker resources ---
+        cls.redis_container = f"{cls.prefix}-redis"
+        cls.ignored_volume = f"{cls.prefix}-redis-vol"
+        cls.normal_volume = f"{cls.prefix}-files-vol"
+
+        cls.containers = [cls.redis_container]
+        cls.volumes = [cls.ignored_volume, cls.normal_volume]
+
+        # Create volumes
+        run(["docker", "volume", "create", cls.ignored_volume])
+        run(["docker", "volume", "create", cls.normal_volume])
+
+        # Start redis container using the ignored volume
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.redis_container,
+                "-v",
+                f"{cls.ignored_volume}:/data",
+                "redis:alpine",
+            ]
+        )
+
+        # Put deterministic content into the normal volume
+        run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "-v",
+                f"{cls.normal_volume}:/data",
+                "alpine:3.20",
+                "sh",
+                "-lc",
+                "mkdir -p /data && echo 'hello' > /data/hello.txt",
+            ]
+        )
+
+        # databases.csv required by CLI (can be empty)
+        cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
+        write_databases_csv(cls.databases_csv, [])
+
+        # Run baudolo with images-no-backup-required redis
+        cmd = [
+            "baudolo",
+            "--compose-dir",
+            cls.compose_dir,
+            "--docker-compose-hard-restart-required",
+            "mailu",
+            "--repo-name",
+            cls.repo_name,
+            "--databases-csv",
+            cls.databases_csv,
+            "--backups-dir",
+            cls.backups_dir,
+            "--database-containers",
+            "dummy-db",
+            "--images-no-stop-required",
+            "alpine",
+            "redis",
+            "postgres",
+            "mariadb",
+            "mysql",
+            "--images-no-backup-required",
+            "redis",
+        ]
+        cp = run(cmd, capture=True, check=True)
+        cls.stdout = cp.stdout or ""
+        cls.stderr = cp.stderr or ""
+
+        cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_ignored_volume_has_no_backup_directory_at_all(self) -> None:
+        p = backup_path(
+            self.backups_dir,
+            self.repo_name,
+            self.version,
+            self.ignored_volume,
+        )
+        self.assertFalse(
+            p.exists(),
+            f"Expected NO backup directory to be created for ignored volume, but found: {p}",
+        )
+
+    def test_normal_volume_is_still_backed_up(self) -> None:
+        p = (
+            backup_path(
+                self.backups_dir,
+                self.repo_name,
+                self.version,
+                self.normal_volume,
+            )
+            / "files"
+            / "hello.txt"
+        )
+        self.assertTrue(p.is_file(), f"Expected backed up file at: {p}")

tests/e2e/test_e2e_mariadb_full.py

@@ -0,0 +1,166 @@
+# tests/e2e/test_e2e_mariadb_full.py
+import unittest
+
+from .helpers import (
+    backup_run,
+    backup_path,
+    cleanup_docker,
+    create_minimal_compose_dir,
+    ensure_empty_dir,
+    latest_version_dir,
+    require_docker,
+    unique,
+    write_databases_csv,
+    run,
+    wait_for_mariadb,
+    wait_for_mariadb_sql,
+)
+
+
+class TestE2EMariaDBFull(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+        cls.prefix = unique("baudolo-e2e-mariadb-full")
+        cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
+        ensure_empty_dir(cls.backups_dir)
+        cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
+        cls.repo_name = cls.prefix
+
+        cls.db_container = f"{cls.prefix}-mariadb"
+        cls.db_volume = f"{cls.prefix}-mariadb-vol"
+        cls.containers = [cls.db_container]
+        cls.volumes = [cls.db_volume]
+
+        cls.db_name = "appdb"
+        cls.db_user = "test"
+        cls.db_password = "testpw"
+        cls.root_password = "rootpw"
+
+        run(["docker", "volume", "create", cls.db_volume])
+
+        # Start MariaDB with a dedicated TCP-capable user for tests.
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.db_container,
+                "-e",
+                f"MARIADB_ROOT_PASSWORD={cls.root_password}",
+                "-e",
+                f"MARIADB_DATABASE={cls.db_name}",
+                "-e",
+                f"MARIADB_USER={cls.db_user}",
+                "-e",
+                f"MARIADB_PASSWORD={cls.db_password}",
+                "-v",
+                f"{cls.db_volume}:/var/lib/mysql",
+                "mariadb:11",
+            ]
+        )
+
+        # Liveness + actual SQL login readiness (TCP)
+        wait_for_mariadb(
+            cls.db_container, root_password=cls.root_password, timeout_s=90
+        )
+        wait_for_mariadb_sql(
+            cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
+        )
+
+        # Create table + data via the dedicated user (TCP)
+        run(
+            [
+                "docker",
+                "exec",
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
+                f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
+            ]
+        )
+
+        cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
+        # IMPORTANT: baudolo backup expects credentials for the DB dump.
+        write_databases_csv(
+            cls.databases_csv,
+            [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)],
+        )
+
+        # Backup with file+dump
+        backup_run(
+            backups_dir=cls.backups_dir,
+            repo_name=cls.repo_name,
+            compose_dir=cls.compose_dir,
+            databases_csv=cls.databases_csv,
+            database_containers=[cls.db_container],
+            images_no_stop_required=["mariadb", "mysql", "alpine", "postgres"],
+        )
+
+        cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
+
+        # Wipe DB via the dedicated user (TCP)
+        run(
+            [
+                "docker",
+                "exec",
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f'-e "DROP TABLE {cls.db_name}.t;"',
+            ]
+        )
+
+        # Restore DB (uses baudolo-restore which execs mysql/mariadb inside the container)
+        run(
+            [
+                "baudolo-restore",
+                "mariadb",
+                cls.db_volume,
+                cls.hash,
+                cls.version,
+                "--backups-dir",
+                cls.backups_dir,
+                "--repo-name",
+                cls.repo_name,
+                "--container",
+                cls.db_container,
+                "--db-name",
+                cls.db_name,
+                "--db-user",
+                cls.db_user,
+                "--db-password",
+                cls.db_password,
+                "--empty",
+            ]
+        )
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_dump_file_exists(self) -> None:
+        p = (
+            backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume)
+            / "sql"
+            / f"{self.db_name}.backup.sql"
+        )
+        self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
+
+    def test_data_restored(self) -> None:
+        p = run(
+            [
+                "docker",
+                "exec",
+                self.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
+                f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
+            ]
+        )
+        self.assertEqual((p.stdout or "").strip(), "ok")

tests/e2e/test_e2e_mariadb_no_copy.py

@@ -0,0 +1,163 @@
+# tests/e2e/test_e2e_mariadb_no_copy.py
+import unittest
+
+from .helpers import (
+    backup_run,
+    backup_path,
+    cleanup_docker,
+    create_minimal_compose_dir,
+    ensure_empty_dir,
+    latest_version_dir,
+    require_docker,
+    unique,
+    write_databases_csv,
+    run,
+    wait_for_mariadb,
+    wait_for_mariadb_sql,
+)
+
+
+class TestE2EMariaDBNoCopy(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+        cls.prefix = unique("baudolo-e2e-mariadb-nocopy")
+        cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
+        ensure_empty_dir(cls.backups_dir)
+        cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
+        cls.repo_name = cls.prefix
+
+        cls.db_container = f"{cls.prefix}-mariadb"
+        cls.db_volume = f"{cls.prefix}-mariadb-vol"
+        cls.containers = [cls.db_container]
+        cls.volumes = [cls.db_volume]
+
+        cls.db_name = "appdb"
+        cls.db_user = "test"
+        cls.db_password = "testpw"
+        cls.root_password = "rootpw"
+
+        run(["docker", "volume", "create", cls.db_volume])
+
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.db_container,
+                "-e",
+                f"MARIADB_ROOT_PASSWORD={cls.root_password}",
+                "-e",
+                f"MARIADB_DATABASE={cls.db_name}",
+                "-e",
+                f"MARIADB_USER={cls.db_user}",
+                "-e",
+                f"MARIADB_PASSWORD={cls.db_password}",
+                "-v",
+                f"{cls.db_volume}:/var/lib/mysql",
+                "mariadb:11",
+            ]
+        )
+
+        wait_for_mariadb(
+            cls.db_container, root_password=cls.root_password, timeout_s=90
+        )
+        wait_for_mariadb_sql(
+            cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
+        )
+
+        # Create table + data (TCP)
+        run(
+            [
+                "docker",
+                "exec",
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
+                f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
+            ]
+        )
+
+        cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
+        write_databases_csv(
+            cls.databases_csv,
+            [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)],
+        )
+
+        # dump-only-sql => no files
+        backup_run(
+            backups_dir=cls.backups_dir,
+            repo_name=cls.repo_name,
+            compose_dir=cls.compose_dir,
+            databases_csv=cls.databases_csv,
+            database_containers=[cls.db_container],
+            images_no_stop_required=["mariadb", "mysql", "alpine", "postgres"],
+            dump_only_sql=True,
+        )
+
+        cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
+
+        # Wipe table (TCP)
+        run(
+            [
+                "docker",
+                "exec",
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f'-e "DROP TABLE {cls.db_name}.t;"',
+            ]
+        )
+
+        # Restore DB
+        run(
+            [
+                "baudolo-restore",
+                "mariadb",
+                cls.db_volume,
+                cls.hash,
+                cls.version,
+                "--backups-dir",
+                cls.backups_dir,
+                "--repo-name",
+                cls.repo_name,
+                "--container",
+                cls.db_container,
+                "--db-name",
+                cls.db_name,
+                "--db-user",
+                cls.db_user,
+                "--db-password",
+                cls.db_password,
+                "--empty",
+            ]
+        )
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_files_backup_not_present(self) -> None:
+        p = (
+            backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume)
+            / "files"
+        )
+        self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
+
+    def test_data_restored(self) -> None:
+        p = run(
+            [
+                "docker",
+                "exec",
+                self.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
+                f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
+            ]
+        )
+        self.assertEqual((p.stdout or "").strip(), "ok")

tests/e2e/test_e2e_postgres_full.py

@@ -0,0 +1,143 @@
+# tests/e2e/test_e2e_postgres_full.py
+import unittest
+
+from .helpers import (
+    backup_run,
+    backup_path,
+    cleanup_docker,
+    create_minimal_compose_dir,
+    ensure_empty_dir,
+    latest_version_dir,
+    require_docker,
+    unique,
+    write_databases_csv,
+    run,
+    wait_for_postgres,
+)
+
+
+class TestE2EPostgresFull(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+        cls.prefix = unique("baudolo-e2e-postgres-full")
+        cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
+        ensure_empty_dir(cls.backups_dir)
+        cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
+        cls.repo_name = cls.prefix
+
+        cls.pg_container = f"{cls.prefix}-pg"
+        cls.pg_volume = f"{cls.prefix}-pg-vol"
+        cls.containers = [cls.pg_container]
+        cls.volumes = [cls.pg_volume]
+
+        run(["docker", "volume", "create", cls.pg_volume])
+
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.pg_container,
+                "-e",
+                "POSTGRES_PASSWORD=pgpw",
+                "-e",
+                "POSTGRES_DB=appdb",
+                "-e",
+                "POSTGRES_USER=postgres",
+                "-v",
+                f"{cls.pg_volume}:/var/lib/postgresql/data",
+                "postgres:16",
+            ]
+        )
+        wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
+
+        # Create a table + data
+        run(
+            [
+                "docker",
+                "exec",
+                cls.pg_container,
+                "sh",
+                "-lc",
+                "psql -U postgres -d appdb -c \"CREATE TABLE t (id int primary key, v text); INSERT INTO t VALUES (1,'ok');\"",
+            ]
+        )
+
+        cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
+        write_databases_csv(
+            cls.databases_csv, [(cls.pg_container, "appdb", "postgres", "pgpw")]
+        )
+
+        backup_run(
+            backups_dir=cls.backups_dir,
+            repo_name=cls.repo_name,
+            compose_dir=cls.compose_dir,
+            databases_csv=cls.databases_csv,
+            database_containers=[cls.pg_container],
+            images_no_stop_required=["postgres", "mariadb", "mysql", "alpine"],
+        )
+
+        cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
+
+        # Wipe schema
+        run(
+            [
+                "docker",
+                "exec",
+                cls.pg_container,
+                "sh",
+                "-lc",
+                'psql -U postgres -d appdb -c "DROP TABLE t;"',
+            ]
+        )
+
+        # Restore
+        run(
+            [
+                "baudolo-restore",
+                "postgres",
+                cls.pg_volume,
+                cls.hash,
+                cls.version,
+                "--backups-dir",
+                cls.backups_dir,
+                "--repo-name",
+                cls.repo_name,
+                "--container",
+                cls.pg_container,
+                "--db-name",
+                "appdb",
+                "--db-user",
+                "postgres",
+                "--db-password",
+                "pgpw",
+                "--empty",
+            ]
+        )
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_dump_file_exists(self) -> None:
+        p = (
+            backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume)
+            / "sql"
+            / "appdb.backup.sql"
+        )
+        self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
+
+    def test_data_restored(self) -> None:
+        p = run(
+            [
+                "docker",
+                "exec",
+                self.pg_container,
+                "sh",
+                "-lc",
+                'psql -U postgres -d appdb -t -c "SELECT v FROM t WHERE id=1;"',
+            ]
+        )
+        self.assertEqual((p.stdout or "").strip(), "ok")

tests/e2e/test_e2e_postgres_no_copy.py

@@ -0,0 +1,139 @@
+# tests/e2e/test_e2e_postgres_no_copy.py
+import unittest
+
+from .helpers import (
+    backup_run,
+    backup_path,
+    cleanup_docker,
+    create_minimal_compose_dir,
+    ensure_empty_dir,
+    latest_version_dir,
+    require_docker,
+    unique,
+    write_databases_csv,
+    run,
+    wait_for_postgres,
+)
+
+
+class TestE2EPostgresNoCopy(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+        cls.prefix = unique("baudolo-e2e-postgres-nocopy")
+        cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
+        ensure_empty_dir(cls.backups_dir)
+        cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
+        cls.repo_name = cls.prefix
+
+        cls.pg_container = f"{cls.prefix}-pg"
+        cls.pg_volume = f"{cls.prefix}-pg-vol"
+        cls.containers = [cls.pg_container]
+        cls.volumes = [cls.pg_volume]
+
+        run(["docker", "volume", "create", cls.pg_volume])
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.pg_container,
+                "-e",
+                "POSTGRES_PASSWORD=pgpw",
+                "-e",
+                "POSTGRES_DB=appdb",
+                "-e",
+                "POSTGRES_USER=postgres",
+                "-v",
+                f"{cls.pg_volume}:/var/lib/postgresql/data",
+                "postgres:16",
+            ]
+        )
+        wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
+
+        run(
+            [
+                "docker",
+                "exec",
+                cls.pg_container,
+                "sh",
+                "-lc",
+                "psql -U postgres -d appdb -c \"CREATE TABLE t (id int primary key, v text); INSERT INTO t VALUES (1,'ok');\"",
+            ]
+        )
+
+        cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
+        write_databases_csv(
+            cls.databases_csv, [(cls.pg_container, "appdb", "postgres", "pgpw")]
+        )
+
+        backup_run(
+            backups_dir=cls.backups_dir,
+            repo_name=cls.repo_name,
+            compose_dir=cls.compose_dir,
+            databases_csv=cls.databases_csv,
+            database_containers=[cls.pg_container],
+            images_no_stop_required=["postgres", "mariadb", "mysql", "alpine"],
+            dump_only_sql=True,
+        )
+
+        cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
+
+        run(
+            [
+                "docker",
+                "exec",
+                cls.pg_container,
+                "sh",
+                "-lc",
+                'psql -U postgres -d appdb -c "DROP TABLE t;"',
+            ]
+        )
+
+        run(
+            [
+                "baudolo-restore",
+                "postgres",
+                cls.pg_volume,
+                cls.hash,
+                cls.version,
+                "--backups-dir",
+                cls.backups_dir,
+                "--repo-name",
+                cls.repo_name,
+                "--container",
+                cls.pg_container,
+                "--db-name",
+                "appdb",
+                "--db-user",
+                "postgres",
+                "--db-password",
+                "pgpw",
+                "--empty",
+            ]
+        )
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_files_backup_not_present(self) -> None:
+        p = (
+            backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume)
+            / "files"
+        )
+        self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
+
+    def test_data_restored(self) -> None:
+        p = run(
+            [
+                "docker",
+                "exec",
+                self.pg_container,
+                "sh",
+                "-lc",
+                'psql -U postgres -d appdb -t -c "SELECT v FROM t WHERE id=1;"',
+            ]
+        )
+        self.assertEqual((p.stdout or "").strip(), "ok")

tests/e2e/test_e2e_seed_star_and_db_entries_backup_postgres.py

@@ -0,0 +1,217 @@
+import unittest
+
+from .helpers import (
+    backup_path,
+    cleanup_docker,
+    create_minimal_compose_dir,
+    ensure_empty_dir,
+    latest_version_dir,
+    require_docker,
+    run,
+    unique,
+    wait_for_postgres,
+)
+
+
+class TestE2ESeedStarAndDbEntriesBackupPostgres(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+
+        cls.prefix = unique("baudolo-e2e-seed-star-and-db")
+        cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
+        ensure_empty_dir(cls.backups_dir)
+
+        cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
+        cls.repo_name = cls.prefix
+
+        # --- Volumes ---
+        cls.db_volume = f"{cls.prefix}-vol-db"
+        cls.files_volume = f"{cls.prefix}-vol-files"
+        cls.volumes = [cls.db_volume, cls.files_volume]
+
+        run(["docker", "volume", "create", cls.db_volume])
+        run(["docker", "volume", "create", cls.files_volume])
+
+        # Put a marker into the non-db volume
+        cls.marker = "hello-non-db-seed-star"
+        run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "-v",
+                f"{cls.files_volume}:/data",
+                "alpine:3.20",
+                "sh",
+                "-lc",
+                f"echo '{cls.marker}' > /data/hello.txt",
+            ]
+        )
+
+        # --- Start Postgres container using the DB volume ---
+        cls.pg_container = f"{cls.prefix}-pg"
+        cls.containers = [cls.pg_container]
+
+        cls.pg_password = "postgres"
+        cls.pg_user = "postgres"
+
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.pg_container,
+                "-e",
+                f"POSTGRES_PASSWORD={cls.pg_password}",
+                "-v",
+                f"{cls.db_volume}:/var/lib/postgresql/data",
+                "postgres:16-alpine",
+            ]
+        )
+        wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
+
+        # Create two DBs and deterministic content, so pg_dumpall is meaningful
+        cls.pg_db1 = "testdb1"
+        cls.pg_db2 = "testdb2"
+
+        run(
+            [
+                "docker",
+                "exec",
+                cls.pg_container,
+                "sh",
+                "-lc",
+                (
+                    f'psql -U {cls.pg_user} -c "CREATE DATABASE {cls.pg_db1};" || true; '
+                    f'psql -U {cls.pg_user} -c "CREATE DATABASE {cls.pg_db2};" || true; '
+                ),
+            ],
+            check=True,
+        )
+
+        run(
+            [
+                "docker",
+                "exec",
+                cls.pg_container,
+                "sh",
+                "-lc",
+                (
+                    f'psql -U {cls.pg_user} -d {cls.pg_db1} -c '
+                    '"CREATE TABLE IF NOT EXISTS t (id INT PRIMARY KEY, v TEXT);'
+                    "INSERT INTO t(id,v) VALUES (1,'hello-db1') "
+                    "ON CONFLICT (id) DO UPDATE SET v=EXCLUDED.v;\""
+                ),
+            ],
+            check=True,
+        )
+        run(
+            [
+                "docker",
+                "exec",
+                cls.pg_container,
+                "sh",
+                "-lc",
+                (
+                    f'psql -U {cls.pg_user} -d {cls.pg_db2} -c '
+                    '"CREATE TABLE IF NOT EXISTS t (id INT PRIMARY KEY, v TEXT);'
+                    "INSERT INTO t(id,v) VALUES (1,'hello-db2') "
+                    "ON CONFLICT (id) DO UPDATE SET v=EXCLUDED.v;\""
+                ),
+            ],
+            check=True,
+        )
+
+        # --- Seed databases.csv using CLI (star + concrete db) ---
+        cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
+
+        # IMPORTANT: because we pass --database-containers <container>,
+        # get_instance() will use the container name as instance key.
+        instance = cls.pg_container
+
+        # Seed star entry (pg_dumpall)
+        run(["baudolo-seed", cls.databases_csv, instance, "*", cls.pg_user, cls.pg_password])
+
+        # Seed concrete DB entry (pg_dump)
+        run(
+            [
+                "baudolo-seed",
+                cls.databases_csv,
+                instance,
+                cls.pg_db1,
+                cls.pg_user,
+                cls.pg_password,
+            ]
+        )
+
+        # --- Run baudolo with dump-only-sql ---
+        cmd = [
+            "baudolo",
+            "--compose-dir",
+            cls.compose_dir,
+            "--databases-csv",
+            cls.databases_csv,
+            "--database-containers",
+            cls.pg_container,
+            "--images-no-stop-required",
+            "alpine",
+            "postgres",
+            "mariadb",
+            "mysql",
+            "--dump-only-sql",
+            "--backups-dir",
+            cls.backups_dir,
+            "--repo-name",
+            cls.repo_name,
+        ]
+        cp = run(cmd, capture=True, check=True)
+        cls.stdout = cp.stdout or ""
+        cls.stderr = cp.stderr or ""
+
+        cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_db_volume_has_cluster_dump_and_concrete_db_dump_and_no_files(self) -> None:
+        base = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume)
+        sql_dir = base / "sql"
+        files_dir = base / "files"
+
+        self.assertTrue(sql_dir.exists(), f"Expected sql dir at: {sql_dir}")
+        self.assertFalse(
+            files_dir.exists(),
+            f"Did not expect files dir for DB volume when dump-only-sql succeeded: {files_dir}",
+        )
+
+        # Cluster dump file produced by '*' entry
+        cluster = sql_dir / f"{self.pg_container}.cluster.backup.sql"
+        self.assertTrue(cluster.is_file(), f"Expected cluster dump file at: {cluster}")
+
+        # Concrete DB dump produced by normal entry
+        db1 = sql_dir / f"{self.pg_db1}.backup.sql"
+        self.assertTrue(db1.is_file(), f"Expected db dump file at: {db1}")
+
+        # Basic sanity: cluster dump usually contains CREATE DATABASE statements
+        txt = cluster.read_text(encoding="utf-8", errors="ignore")
+        self.assertIn(
+            "CREATE DATABASE",
+            txt,
+            "Expected cluster dump to contain CREATE DATABASE statements",
+        )
+
+    def test_non_db_volume_still_has_files_backup(self) -> None:
+        base = backup_path(self.backups_dir, self.repo_name, self.version, self.files_volume)
+        files_dir = base / "files"
+
+        self.assertTrue(files_dir.exists(), f"Expected files dir for non-DB volume at: {files_dir}")
+
+        marker = files_dir / "hello.txt"
+        self.assertTrue(marker.is_file(), f"Expected marker file at: {marker}")
+        self.assertEqual(
+            marker.read_text(encoding="utf-8").strip(),
+            self.marker,
+        )

tests/integration/test_seed_integration.py

@@ -0,0 +1,213 @@
+import csv
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+def run_seed(
+    csv_path: Path, instance: str, database: str, username: str, password: str
+) -> subprocess.CompletedProcess:
+    """
+    Run the real CLI module (E2E-style) using subprocess.
+
+    Seed contract (current):
+    - database must be "*" or a valid name (non-empty, matches allowed charset)
+    - password is required
+    - entry is keyed by (instance, database); username/password get updated
+    """
+    cp = subprocess.run(
+        [
+            sys.executable,
+            "-m",
+            "baudolo.seed",
+            str(csv_path),
+            instance,
+            database,
+            username,
+            password,
+        ],
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    if cp.returncode != 0:
+        raise AssertionError(
+            "seed command failed unexpectedly.\n"
+            f"returncode: {cp.returncode}\n"
+            f"stdout:\n{cp.stdout}\n"
+            f"stderr:\n{cp.stderr}\n"
+        )
+    return cp
+
+
+def run_seed_expect_fail(
+    csv_path: Path, instance: str, database: str, username: str, password: str
+) -> subprocess.CompletedProcess:
+    """
+    Same as run_seed, but expects non-zero exit. Returns CompletedProcess for inspection.
+    """
+    return subprocess.run(
+        [
+            sys.executable,
+            "-m",
+            "baudolo.seed",
+            str(csv_path),
+            instance,
+            database,
+            username,
+            password,
+        ],
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+
+
+def read_csv_semicolon(path: Path) -> list[dict]:
+    with path.open("r", encoding="utf-8", newline="") as f:
+        reader = csv.DictReader(f, delimiter=";")
+        return list(reader)
+
+
+def read_text(path: Path) -> str:
+    return path.read_text(encoding="utf-8")
+
+
+class TestSeedIntegration(unittest.TestCase):
+    def test_creates_file_and_adds_entry_when_missing(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            p = Path(td) / "databases.csv"
+            self.assertFalse(p.exists())
+
+            cp = run_seed(p, "docker.test", "appdb", "alice", "secret")
+
+            self.assertEqual(cp.returncode, 0)
+            self.assertTrue(p.exists())
+
+            rows = read_csv_semicolon(p)
+            self.assertEqual(len(rows), 1)
+            self.assertEqual(rows[0]["instance"], "docker.test")
+            self.assertEqual(rows[0]["database"], "appdb")
+            self.assertEqual(rows[0]["username"], "alice")
+            self.assertEqual(rows[0]["password"], "secret")
+
+    def test_replaces_existing_entry_same_instance_and_database_updates_username_and_password(
+        self,
+    ) -> None:
+        """
+        Replacement semantics:
+        - Key is (instance, database)
+        - username/password are updated in-place
+        """
+        with tempfile.TemporaryDirectory() as td:
+            p = Path(td) / "databases.csv"
+
+            run_seed(p, "docker.test", "appdb", "alice", "oldpw")
+            rows = read_csv_semicolon(p)
+            self.assertEqual(len(rows), 1)
+            self.assertEqual(rows[0]["username"], "alice")
+            self.assertEqual(rows[0]["password"], "oldpw")
+
+            run_seed(p, "docker.test", "appdb", "bob", "newpw")
+            rows = read_csv_semicolon(p)
+
+            self.assertEqual(len(rows), 1, "Expected replacement, not a duplicate row")
+            self.assertEqual(rows[0]["instance"], "docker.test")
+            self.assertEqual(rows[0]["database"], "appdb")
+            self.assertEqual(rows[0]["username"], "bob")
+            self.assertEqual(rows[0]["password"], "newpw")
+
+    def test_allows_star_database_for_dump_all(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            p = Path(td) / "databases.csv"
+
+            cp = run_seed(p, "bigbluebutton", "*", "postgres", "pw")
+            self.assertEqual(cp.returncode, 0)
+
+            rows = read_csv_semicolon(p)
+            self.assertEqual(len(rows), 1)
+            self.assertEqual(rows[0]["instance"], "bigbluebutton")
+            self.assertEqual(rows[0]["database"], "*")
+            self.assertEqual(rows[0]["username"], "postgres")
+            self.assertEqual(rows[0]["password"], "pw")
+
+    def test_replaces_existing_star_entry(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            p = Path(td) / "databases.csv"
+
+            run_seed(p, "bigbluebutton", "*", "postgres", "pw1")
+            run_seed(p, "bigbluebutton", "*", "postgres", "pw2")
+
+            rows = read_csv_semicolon(p)
+            self.assertEqual(len(rows), 1)
+            self.assertEqual(rows[0]["database"], "*")
+            self.assertEqual(rows[0]["password"], "pw2")
+
+    def test_rejects_empty_database_value(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            p = Path(td) / "databases.csv"
+
+            cp = run_seed_expect_fail(p, "docker.test", "", "alice", "pw")
+            self.assertNotEqual(cp.returncode, 0)
+
+            combined = ((cp.stdout or "") + "\n" + (cp.stderr or "")).lower()
+            self.assertIn("error:", combined)
+            self.assertIn("database", combined)
+            self.assertIn("not empty", combined)
+
+            self.assertFalse(p.exists(), "Should not create file on invalid input")
+
+    def test_rejects_invalid_database_name_characters(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            p = Path(td) / "databases.csv"
+
+            cp = run_seed_expect_fail(p, "docker.test", "app db", "alice", "pw")
+            self.assertNotEqual(cp.returncode, 0)
+
+            combined = ((cp.stdout or "") + "\n" + (cp.stderr or "")).lower()
+            self.assertIn("error:", combined)
+            self.assertIn("invalid database name", combined)
+
+            self.assertFalse(p.exists(), "Should not create file on invalid input")
+
+    def test_rejects_nan_database_name(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            p = Path(td) / "databases.csv"
+
+            cp = run_seed_expect_fail(p, "docker.test", "nan", "alice", "pw")
+            self.assertNotEqual(cp.returncode, 0)
+
+            combined = ((cp.stdout or "") + "\n" + (cp.stderr or "")).lower()
+            self.assertIn("error:", combined)
+            self.assertIn("must not be 'nan'", combined)
+
+            self.assertFalse(p.exists(), "Should not create file on invalid input")
+
+    def test_accepts_hyphen_and_underscore_database_names(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            p = Path(td) / "databases.csv"
+
+            run_seed(p, "docker.test", "my_db-1", "alice", "pw")
+
+            rows = read_csv_semicolon(p)
+            self.assertEqual(len(rows), 1)
+            self.assertEqual(rows[0]["database"], "my_db-1")
+
+    def test_file_is_semicolon_delimited_and_has_header(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            p = Path(td) / "databases.csv"
+
+            run_seed(p, "docker.test", "appdb", "alice", "pw")
+
+            txt = read_text(p)
+            self.assertTrue(
+                txt.startswith("instance;database;username;password"),
+                f"Unexpected header / delimiter in file:\n{txt}",
+            )
+            self.assertIn(";", txt)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/test_backup.py

@@ -1,64 +1,40 @@
-# tests/unit/test_backup.py
-
 import unittest
 from unittest.mock import patch
-import importlib.util
-import sys
-import os
-import pathlib
 
-# Prevent actual directory creation in backup script import
+from baudolo.backup.app import requires_stop
-dummy_mkdir = lambda self, *args, **kwargs: None
-original_mkdir = pathlib.Path.mkdir
-pathlib.Path.mkdir = dummy_mkdir
 
-# Create a virtual databases.csv in the project root for the module import
-test_dir = os.path.dirname(__file__)
-project_root = os.path.abspath(os.path.join(test_dir, '../../'))
-sys.path.insert(0, project_root)
-db_csv_path = os.path.join(project_root, 'databases.csv')
-with open(db_csv_path, 'w') as f:
-    f.write('instance;database;username;password\n')
 
-# Dynamically load the hyphenated script as module 'backup'
+class TestRequiresStop(unittest.TestCase):
-script_path = os.path.join(project_root, 'backup-docker-to-local.py')
+    @patch("baudolo.backup.app.get_image_info")
-spec = importlib.util.spec_from_file_location('backup', script_path)
+    def test_requires_stop_false_when_all_images_are_whitelisted(
-backup = importlib.util.module_from_spec(spec)
+        self, mock_get_image_info
-sys.modules['backup'] = backup
+    ):
-spec.loader.exec_module(backup)
+        # All containers use images containing allowed substrings
+        mock_get_image_info.side_effect = [
+            "repo/mastodon:v4",
+            "repo/wordpress:latest",
+        ]
+        containers = ["c1", "c2"]
+        whitelist = ["mastodon", "wordpress"]
+        self.assertFalse(requires_stop(containers, whitelist))
 
-# Restore original mkdir
+    @patch("baudolo.backup.app.get_image_info")
-pathlib.Path.mkdir = original_mkdir
+    def test_requires_stop_true_when_any_image_is_not_whitelisted(
+        self, mock_get_image_info
+    ):
+        mock_get_image_info.side_effect = [
+            "repo/mastodon:v4",
+            "repo/nginx:latest",
+        ]
+        containers = ["c1", "c2"]
+        whitelist = ["mastodon", "wordpress"]
+        self.assertTrue(requires_stop(containers, whitelist))
 
-class TestIsImageWhitelisted(unittest.TestCase):
+    @patch("baudolo.backup.app.get_image_info")
-    @patch('backup.get_image_info')
+    def test_requires_stop_true_when_whitelist_empty(self, mock_get_image_info):
-    def test_returns_true_when_image_matches(self, mock_get_image_info):
+        mock_get_image_info.return_value = "repo/anything:latest"
-        # Simulate a container image containing 'mastodon'
+        self.assertTrue(requires_stop(["c1"], []))
-        mock_get_image_info.return_value = ['repo/mastodon:v4']
-        images = ['mastodon', 'wordpress']
-        self.assertTrue(
-            backup.is_image_whitelisted('any_container', images),
-            "Should return True when at least one image substring matches"
-        )
 
-    @patch('backup.get_image_info')
-    def test_returns_false_when_no_image_matches(self, mock_get_image_info):
-        # Simulate a container image without matching substrings
-        mock_get_image_info.return_value = ['repo/nginx:latest']
-        images = ['mastodon', 'wordpress']
-        self.assertFalse(
-            backup.is_image_whitelisted('any_container', images),
-            "Should return False when no image substring matches"
-        )
 
-    @patch('backup.get_image_info')
+if __name__ == "__main__":
-    def test_returns_false_with_empty_image_list(self, mock_get_image_info):
-        # Even if get_image_info returns something, an empty list yields False
-        mock_get_image_info.return_value = ['repo/element:1.0']
-        self.assertFalse(
-            backup.is_image_whitelisted('any_container', []),
-            "Should return False when the images list is empty"
-        )
-
-if __name__ == '__main__':
     unittest.main()