mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-07-17 06:05:13 +00:00
Compare commits
40 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 45d3b0ad7c | |||
| fe5bed8254 | |||
| 7a7ec57b54 | |||
| 2bbe7d180a | |||
|
|
286ef179da | ||
|
|
6cb0b8a548 | ||
| e1f1b602d3 | |||
| b9a8b391f0 | |||
| c949f2c5cf | |||
| 96e6b3ea93 | |||
| 79214e64e8 | |||
| e9030e8443 | |||
| 57ea4592c1 | |||
| ad5d8fcda3 | |||
| bfa596ae30 | |||
| 21b4d237d3 | |||
| ec051b4c2b | |||
| ed78f69b3b | |||
| a69074c302 | |||
| 0b4696f649 | |||
| e3f28098bd | |||
| babadcb038 | |||
| fbfdb8615f | |||
| 2f5882f5c1 | |||
| 522391fdd3 | |||
| b3c9cf5ce1 | |||
| 2ed3472527 | |||
| 54737cefa7 | |||
| d976640312 | |||
| e4bc075474 | |||
| f3ef86a444 | |||
| c01ab55f2d | |||
| e3cdfd6fc4 | |||
| df32671cec | |||
| d563dce20f | |||
| 0222f7f109 | |||
| 6adafe6b1f | |||
| 88b35ee923 | |||
| 71f79929be | |||
| 0fb8efba4f |
21
.github/dependabot.yml
vendored
Normal file
21
.github/dependabot.yml
vendored
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
---
|
||||||
|
version: 2
|
||||||
|
updates:
|
||||||
|
- package-ecosystem: github-actions
|
||||||
|
directory: /
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
|
groups:
|
||||||
|
actions:
|
||||||
|
patterns:
|
||||||
|
- "*"
|
||||||
|
|
||||||
|
- package-ecosystem: docker
|
||||||
|
directory: /
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
|
|
||||||
|
- package-ecosystem: pip
|
||||||
|
directory: /
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
8
.github/workflows/ci.yml
vendored
8
.github/workflows/ci.yml
vendored
@@ -22,7 +22,7 @@ jobs:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
- name: Show docker info
|
- name: Show docker info
|
||||||
run: |
|
run: |
|
||||||
@@ -35,7 +35,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Upload E2E artifacts (always)
|
- name: Upload E2E artifacts (always)
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v7
|
||||||
with:
|
with:
|
||||||
name: e2e-artifacts
|
name: e2e-artifacts
|
||||||
path: artifacts
|
path: artifacts
|
||||||
@@ -49,7 +49,7 @@ jobs:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout (full history for tags)
|
- name: Checkout (full history for tags)
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
@@ -67,7 +67,7 @@ jobs:
|
|||||||
git push -f origin stable
|
git push -f origin stable
|
||||||
|
|
||||||
- name: Login to GHCR
|
- name: Login to GHCR
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@v4
|
||||||
with:
|
with:
|
||||||
registry: ${{ env.REGISTRY }}
|
registry: ${{ env.REGISTRY }}
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
|
|||||||
29
.github/workflows/dependabot-auto-merge.yml
vendored
Normal file
29
.github/workflows/dependabot-auto-merge.yml
vendored
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
---
|
||||||
|
name: Dependabot auto-merge
|
||||||
|
|
||||||
|
on: pull_request
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
pull-requests: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
auto-merge:
|
||||||
|
# Enable auto-merge for minor/patch dependency bumps; majors stay manual.
|
||||||
|
# The merge only happens once every required status check (make test) is
|
||||||
|
# green, so the CI stays the gate.
|
||||||
|
if: github.actor == 'dependabot[bot]'
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Fetch dependency metadata
|
||||||
|
id: metadata
|
||||||
|
uses: dependabot/fetch-metadata@v3
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Enable auto-merge (minor + patch)
|
||||||
|
if: steps.metadata.outputs.update-type != 'version-update:semver-major'
|
||||||
|
run: gh pr merge --auto --squash "$PR_URL"
|
||||||
|
env:
|
||||||
|
PR_URL: ${{ github.event.pull_request.html_url }}
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
5
.gitignore
vendored
5
.gitignore
vendored
@@ -1,2 +1,5 @@
|
|||||||
__pycache__
|
__pycache__
|
||||||
artifacts/
|
artifacts/
|
||||||
|
*.egg-info
|
||||||
|
dist/
|
||||||
|
build/
|
||||||
92
CHANGELOG.md
92
CHANGELOG.md
@@ -1,3 +1,95 @@
|
|||||||
|
# Changelog
|
||||||
|
|
||||||
|
## [1.8.1] - 2026-07-12
|
||||||
|
|
||||||
|
- Restore: the postgres empty mode also drops user-owned collations in
|
||||||
|
public; dumps containing CREATE COLLATION (e.g. OpenProject's ICU
|
||||||
|
collation versions_name) no longer abort the replay with 'collation
|
||||||
|
already exists'.
|
||||||
|
- Maintenance: base image bumped from python 3.11-slim to 3.14-slim.
|
||||||
|
|
||||||
|
## [1.8.0] - 2026-07-11
|
||||||
|
|
||||||
|
Swarm-aware backups and replayable restores.
|
||||||
|
|
||||||
|
- Backup: swarm task containers are never stopped or started manually
|
||||||
|
anymore; they are skipped visibly and backed up hot, while the sql dump
|
||||||
|
stays the consistent database backup.
|
||||||
|
- Backup: a container that vanishes between listing and inspect no longer
|
||||||
|
aborts the run; a failing inspect on a container that still exists keeps
|
||||||
|
failing loudly.
|
||||||
|
- Backup: pg_dump runs with the no-owner and no-privileges flags so dumps
|
||||||
|
are replayable by the owning app user.
|
||||||
|
- Restore: the mariadb empty mode drops all tables in one client session
|
||||||
|
with FOREIGN_KEY_CHECKS disabled; FK-linked parent tables no longer abort
|
||||||
|
the replay with ERROR 1451.
|
||||||
|
- Restore: the postgres empty mode drops only current-user-owned objects,
|
||||||
|
and the replay skips superuser-only dump lines without ever touching
|
||||||
|
COPY data blocks.
|
||||||
|
- Restore: the replay streams the dump through a temp file instead of
|
||||||
|
buffering it in memory; multi-GB dumps no longer OOM the restore.
|
||||||
|
- Tooling: the e2e runner reaches the DinD daemon via docker exec instead
|
||||||
|
of a host-published unencrypted API port.
|
||||||
|
- Tooling: new end-to-end test reproducing the swarm stop flake, plus unit
|
||||||
|
tests for the restore filters and the swarm probes; the suite is 36 unit,
|
||||||
|
9 integration and 30 e2e tests.
|
||||||
|
- Tooling: Dependabot with auto-merge for minor and patch updates.
|
||||||
|
|
||||||
|
## [1.7.1] - 2026-05-26
|
||||||
|
|
||||||
|
* 🔌 MariaDB SQL backups now connect over TCP loopback so the dump always matches the same wildcard-host grant the application uses — no more surprise `ERROR 1045 Access denied` when a localhost-bound auth row preempts.
|
||||||
|
* 🧪 New regression and bug-repro tests pin the TCP behaviour and prove it under the exact preemption setup that caused the production failure on MariaDB 12.
|
||||||
|
* 🩺 E2E test infrastructure: DinD bridge and inner daemon now default to MTU 1280 so registry pulls survive host paths with broken PMTUD (override via `E2E_DIND_MTU`).
|
||||||
|
|
||||||
|
|
||||||
|
## [1.7.0] - 2026-02-07
|
||||||
|
|
||||||
|
* 🚀 Backup jobs now support all valid Docker Compose file names – case-insensitive and hassle-free.
|
||||||
|
|
||||||
|
|
||||||
|
## [1.6.0] - 2026-02-06
|
||||||
|
|
||||||
|
* Compose handling is now fully delegated to the Infinito.Nexus compose wrapper or plain docker compose, removing all custom env and file detection to ensure a single, consistent source of truth.
|
||||||
|
|
||||||
|
|
||||||
|
## [1.5.0] - 2026-01-31
|
||||||
|
|
||||||
|
* * Make `databases.csv` optional: missing or empty files now emit warnings and no longer break backups
|
||||||
|
* Fix Docker CLI compatibility by switching to `docker-ce-cli` and required build tools
|
||||||
|
|
||||||
|
|
||||||
|
## [1.4.0] - 2026-01-31
|
||||||
|
|
||||||
|
* Baudolo now restarts Docker Compose stacks in a wrapper-aware way (with a `docker compose` fallback), ensuring that all Compose overrides and env files are applied identically to the Infinito.Nexus workflow.
|
||||||
|
|
||||||
|
|
||||||
|
## [1.3.0] - 2026-01-10
|
||||||
|
|
||||||
|
* Empty databases.csv no longer causes baudolo-seed to fail
|
||||||
|
|
||||||
|
|
||||||
|
## [1.2.0] - 2025-12-29
|
||||||
|
|
||||||
|
* * Introduced **`--dump-only-sql`** mode for reliable, SQL-only database backups (replaces `--dump-only`).
|
||||||
|
* Database configuration in `databases.csv` is now **strict and explicit** (`*` or concrete database name only).
|
||||||
|
* **PostgreSQL cluster backups** are supported via `*`.
|
||||||
|
* SQL dumps are written **atomically** to avoid corrupted or empty files.
|
||||||
|
* Backups are **smarter and faster**: ignored volumes are skipped early, file backups run only when needed.
|
||||||
|
* Improved reliability through expanded end-to-end tests and safer defaults.
|
||||||
|
|
||||||
|
|
||||||
|
## [1.1.1] - 2025-12-28
|
||||||
|
|
||||||
|
* * **Backup:** In ***--dump-only-sql*** mode, fall back to file backups with a warning when no database dump can be produced (e.g. missing `databases.csv` entry).
|
||||||
|
|
||||||
|
|
||||||
|
## [1.1.0] - 2025-12-28
|
||||||
|
|
||||||
|
* * **Backup:** Log a warning and skip database dumps when no databases.csv entry is present instead of raising an exception; introduce module-level logging and apply formatting cleanups across backup/restore code and tests.
|
||||||
|
* **CLI:** Switch to an FHS-compliant default backup directory (/var/lib/backup) and use a stable default repository name instead of dynamic detection.
|
||||||
|
* **Maintenance:** Update mirror configuration and ignore generated .egg-info files.
|
||||||
|
|
||||||
|
|
||||||
## [1.0.0] - 2025-12-27
|
## [1.0.0] - 2025-12-27
|
||||||
|
|
||||||
* Official Release 🥳
|
* Official Release 🥳
|
||||||
|
|||||||
35
Dockerfile
35
Dockerfile
@@ -1,34 +1,37 @@
|
|||||||
# syntax=docker/dockerfile:1
|
# syntax=docker/dockerfile:1
|
||||||
FROM python:3.11-slim
|
FROM python:3.14-slim
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Runtime + build essentials:
|
# Base deps for build/runtime + docker repo key
|
||||||
# - rsync: required for file backup/restore
|
|
||||||
# - ca-certificates: TLS
|
|
||||||
# - docker-cli: needed if you want to control the host Docker engine (via /var/run/docker.sock mount)
|
|
||||||
# - make: to delegate install logic to Makefile
|
|
||||||
#
|
|
||||||
# Notes:
|
|
||||||
# - On Debian slim, the docker client package is typically "docker.io".
|
|
||||||
# - If you only want restore-without-docker, you can drop docker.io later.
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
make \
|
make \
|
||||||
rsync \
|
rsync \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
docker-cli \
|
bash \
|
||||||
|
curl \
|
||||||
|
gnupg \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Install Docker CLI (docker-ce-cli) from Docker's official apt repo
|
||||||
|
RUN bash -lc "set -euo pipefail \
|
||||||
|
&& install -m 0755 -d /etc/apt/keyrings \
|
||||||
|
&& curl -fsSL https://download.docker.com/linux/debian/gpg \
|
||||||
|
| gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
|
||||||
|
&& chmod a+r /etc/apt/keyrings/docker.gpg \
|
||||||
|
&& . /etc/os-release \
|
||||||
|
&& echo \"deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \${VERSION_CODENAME} stable\" \
|
||||||
|
> /etc/apt/sources.list.d/docker.list \
|
||||||
|
&& apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends docker-ce-cli \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*"
|
||||||
|
|
||||||
# Fail fast if docker client is missing
|
# Fail fast if docker client is missing
|
||||||
|
RUN docker version || true
|
||||||
RUN command -v docker
|
RUN command -v docker
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# All install decisions are handled by the Makefile.
|
|
||||||
RUN make install
|
RUN make install
|
||||||
|
|
||||||
# Sensible defaults (can be overridden at runtime)
|
|
||||||
ENV PYTHONUNBUFFERED=1
|
ENV PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
# Default: show CLI help
|
|
||||||
CMD ["baudolo", "--help"]
|
CMD ["baudolo", "--help"]
|
||||||
|
|||||||
2
MIRRORS
2
MIRRORS
@@ -1,4 +1,4 @@
|
|||||||
git@github.com:kevinveenbirkenbach/backup-docker-to-local.git
|
git@github.com:kevinveenbirkenbach/backup-docker-to-local.git
|
||||||
ssh://git@git.veen.world:2201/kevinveenbirkenbach/backup-docker-to-local.git
|
ssh://git@git.veen.world:2201/kevinveenbirkenbach/backup-docker-to-local.git
|
||||||
ssh://git@code.infinito.nexus:2201/kevinveenbirkenbach/backup-docker-to-local.git
|
ssh://git@code.infinito.nexus:2201/kevinveenbirkenbach/backup-docker-to-local.git
|
||||||
https://pypi.org/project/baudolo/
|
https://pypi.org/project/backup-docker-to-local/
|
||||||
|
|||||||
4
Makefile
4
Makefile
@@ -49,9 +49,9 @@ test: test-unit test-integration test-e2e
|
|||||||
test-unit: clean build
|
test-unit: clean build
|
||||||
@echo ">> Running unit tests"
|
@echo ">> Running unit tests"
|
||||||
@docker run --rm -t $(IMAGE) \
|
@docker run --rm -t $(IMAGE) \
|
||||||
sh -lc 'python -m unittest discover -t . -s tests/unit -p "test_*.py" -v'
|
bash -lc 'python -m unittest discover -t . -s tests/unit -p "test_*.py" -v'
|
||||||
|
|
||||||
test-integration: clean build
|
test-integration: clean build
|
||||||
@echo ">> Running integration tests"
|
@echo ">> Running integration tests"
|
||||||
@docker run --rm -t $(IMAGE) \
|
@docker run --rm -t $(IMAGE) \
|
||||||
sh -lc 'python -m unittest discover -t . -s tests/integration -p "test_*.py" -v'
|
bash -lc 'python -m unittest discover -t . -s tests/integration -p "test_*.py" -v'
|
||||||
@@ -134,7 +134,7 @@ baudolo \
|
|||||||
| Flag | Description |
|
| Flag | Description |
|
||||||
| --------------- | ------------------------------------------- |
|
| --------------- | ------------------------------------------- |
|
||||||
| `--everything` | Always stop containers and re-run rsync |
|
| `--everything` | Always stop containers and re-run rsync |
|
||||||
| `--dump-only` | Only create SQL dumps, skip file backups |
|
| `--dump-only-sql`| Skip file backups only for DB volumes when dumps succeed; non-DB volumes are still backed up; fallback to files if no dump. |
|
||||||
| `--shutdown` | Do not restart containers after backup |
|
| `--shutdown` | Do not restart containers after backup |
|
||||||
| `--backups-dir` | Backup root directory (default: `/Backups`) |
|
| `--backups-dir` | Backup root directory (default: `/Backups`) |
|
||||||
| `--repo-name` | Backup namespace under machine hash |
|
| `--repo-name` | Backup namespace under machine hash |
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "backup-docker-to-local"
|
name = "backup-docker-to-local"
|
||||||
version = "1.0.0"
|
version = "1.8.1"
|
||||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
|
|||||||
@@ -16,15 +16,24 @@ DIND="${E2E_DIND_NAME:-baudolo-e2e-dind}"
|
|||||||
DIND_VOL="${E2E_DIND_VOL:-baudolo-e2e-dind-data}"
|
DIND_VOL="${E2E_DIND_VOL:-baudolo-e2e-dind-data}"
|
||||||
E2E_TMP_VOL="${E2E_TMP_VOL:-baudolo-e2e-tmp}"
|
E2E_TMP_VOL="${E2E_TMP_VOL:-baudolo-e2e-tmp}"
|
||||||
|
|
||||||
DIND_HOST="${E2E_DIND_HOST:-tcp://127.0.0.1:2375}"
|
# Host-side access to the DinD daemon goes through `docker exec` (dind()
|
||||||
|
# below) instead of a host-published port: port publishing is not reachable
|
||||||
|
# from every environment (sandboxed runners, hosts with broken loopback
|
||||||
|
# publishing), while exec only needs the outer docker socket. The TCP
|
||||||
|
# listener stays for the test container inside the dedicated network.
|
||||||
DIND_HOST_IN_NET="${E2E_DIND_HOST_IN_NET:-tcp://${DIND}:2375}"
|
DIND_HOST_IN_NET="${E2E_DIND_HOST_IN_NET:-tcp://${DIND}:2375}"
|
||||||
|
|
||||||
|
dind() { docker exec "${DIND}" docker "$@"; }
|
||||||
|
dind_stdin() { docker exec -i "${DIND}" docker "$@"; }
|
||||||
|
|
||||||
IMG="${E2E_IMAGE:-baudolo:local}"
|
IMG="${E2E_IMAGE:-baudolo:local}"
|
||||||
RSYNC_IMG="${E2E_RSYNC_IMAGE:-ghcr.io/kevinveenbirkenbach/alpine-rsync}"
|
RSYNC_IMG="${E2E_RSYNC_IMAGE:-ghcr.io/kevinveenbirkenbach/alpine-rsync}"
|
||||||
|
|
||||||
READY_TIMEOUT_SECONDS="${E2E_READY_TIMEOUT_SECONDS:-120}"
|
READY_TIMEOUT_SECONDS="${E2E_READY_TIMEOUT_SECONDS:-120}"
|
||||||
ARTIFACTS_DIR="${E2E_ARTIFACTS_DIR:-./artifacts}"
|
ARTIFACTS_DIR="${E2E_ARTIFACTS_DIR:-./artifacts}"
|
||||||
|
|
||||||
|
DIND_MTU="${E2E_DIND_MTU:-1280}"
|
||||||
|
|
||||||
KEEP_ON_FAIL="${E2E_KEEP_ON_FAIL:-0}"
|
KEEP_ON_FAIL="${E2E_KEEP_ON_FAIL:-0}"
|
||||||
KEEP_VOLUMES="${E2E_KEEP_VOLUMES:-0}"
|
KEEP_VOLUMES="${E2E_KEEP_VOLUMES:-0}"
|
||||||
DEBUG_SHELL="${E2E_DEBUG_SHELL:-0}"
|
DEBUG_SHELL="${E2E_DEBUG_SHELL:-0}"
|
||||||
@@ -46,8 +55,8 @@ dump_debug() {
|
|||||||
echo "=== Host docker info ==="
|
echo "=== Host docker info ==="
|
||||||
docker info || true
|
docker info || true
|
||||||
echo
|
echo
|
||||||
echo "=== DinD reachable? (docker -H ${DIND_HOST} version) ==="
|
echo "=== DinD reachable? (docker exec ${DIND} docker version) ==="
|
||||||
docker -H "${DIND_HOST}" version || true
|
dind version || true
|
||||||
echo
|
echo
|
||||||
} > "${ARTIFACTS_DIR}/debug-host-${TS}.txt" 2>&1 || true
|
} > "${ARTIFACTS_DIR}/debug-host-${TS}.txt" 2>&1 || true
|
||||||
|
|
||||||
@@ -56,48 +65,34 @@ dump_debug() {
|
|||||||
|
|
||||||
# DinD state
|
# DinD state
|
||||||
{
|
{
|
||||||
echo "=== docker -H ps -a ==="
|
echo "=== dind ps -a ==="
|
||||||
docker -H "${DIND_HOST}" ps -a || true
|
dind ps -a || true
|
||||||
echo
|
echo
|
||||||
echo "=== docker -H images ==="
|
echo "=== dind images ==="
|
||||||
docker -H "${DIND_HOST}" images || true
|
dind images || true
|
||||||
echo
|
echo
|
||||||
echo "=== docker -H network ls ==="
|
echo "=== dind network ls ==="
|
||||||
docker -H "${DIND_HOST}" network ls || true
|
dind network ls || true
|
||||||
echo
|
echo
|
||||||
echo "=== docker -H volume ls ==="
|
echo "=== dind volume ls ==="
|
||||||
docker -H "${DIND_HOST}" volume ls || true
|
dind volume ls || true
|
||||||
echo
|
echo
|
||||||
echo "=== docker -H system df ==="
|
echo "=== dind system df ==="
|
||||||
docker -H "${DIND_HOST}" system df || true
|
dind system df || true
|
||||||
} > "${ARTIFACTS_DIR}/debug-dind-${TS}.txt" 2>&1 || true
|
} > "${ARTIFACTS_DIR}/debug-dind-${TS}.txt" 2>&1 || true
|
||||||
|
|
||||||
# Try to capture recent events (best effort; might be noisy)
|
# Try to capture recent events (best effort; might be noisy)
|
||||||
docker -H "${DIND_HOST}" events --since 10m --until 0s \
|
dind events --since 10m --until 0s \
|
||||||
> "${ARTIFACTS_DIR}/dind-events-${TS}.txt" 2>&1 || true
|
> "${ARTIFACTS_DIR}/dind-events-${TS}.txt" 2>&1 || true
|
||||||
|
|
||||||
# Dump shared /tmp content from the tmp volume:
|
# The shared tmp volume is mounted at /tmp inside the DinD container
|
||||||
# We create a temporary container that mounts the volume, then tar its content.
|
# itself, so tar it there and copy it out with the outer daemon.
|
||||||
# (Does not rely on host filesystem paths.)
|
|
||||||
log "DEBUG: archiving shared /tmp (volume ${E2E_TMP_VOL})"
|
log "DEBUG: archiving shared /tmp (volume ${E2E_TMP_VOL})"
|
||||||
docker -H "${DIND_HOST}" run --rm \
|
docker exec "${DIND}" tar -czf "/tmpdump-${TS}.tar.gz" -C /tmp . >/dev/null 2>&1 || true
|
||||||
-v "${E2E_TMP_VOL}:/tmp" \
|
docker cp "${DIND}:/tmpdump-${TS}.tar.gz" "${ARTIFACTS_DIR}/e2e-tmp-${TS}.tar.gz" >/dev/null 2>&1 || true
|
||||||
alpine:3.20 \
|
|
||||||
sh -lc 'cd /tmp && tar -czf /out.tar.gz . || true' \
|
|
||||||
>/dev/null 2>&1 || true
|
|
||||||
|
|
||||||
# The above writes inside the container FS, not to host. So do it properly:
|
|
||||||
# Use "docker cp" from a temp container.
|
|
||||||
local tmpc="baudolo-e2e-tmpdump-${TS}"
|
|
||||||
docker -H "${DIND_HOST}" rm -f "${tmpc}" >/dev/null 2>&1 || true
|
|
||||||
docker -H "${DIND_HOST}" create --name "${tmpc}" -v "${E2E_TMP_VOL}:/tmp" alpine:3.20 \
|
|
||||||
sh -lc 'cd /tmp && tar -czf /tmpdump.tar.gz . || true' >/dev/null
|
|
||||||
docker -H "${DIND_HOST}" start -a "${tmpc}" >/dev/null 2>&1 || true
|
|
||||||
docker -H "${DIND_HOST}" cp "${tmpc}:/tmpdump.tar.gz" "${ARTIFACTS_DIR}/e2e-tmp-${TS}.tar.gz" >/dev/null 2>&1 || true
|
|
||||||
docker -H "${DIND_HOST}" rm -f "${tmpc}" >/dev/null 2>&1 || true
|
|
||||||
|
|
||||||
log "DEBUG: artifacts written:"
|
log "DEBUG: artifacts written:"
|
||||||
ls -la "${ARTIFACTS_DIR}" | sed 's/^/ /' || true
|
find "${ARTIFACTS_DIR}" -maxdepth 1 -mindepth 1 -print | sed 's/^/ /' || true
|
||||||
}
|
}
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
@@ -105,9 +100,9 @@ cleanup() {
|
|||||||
log "KEEP_ON_FAIL=1 and failure detected -> skipping cleanup."
|
log "KEEP_ON_FAIL=1 and failure detected -> skipping cleanup."
|
||||||
log "Next steps:"
|
log "Next steps:"
|
||||||
echo " - Inspect DinD logs: docker logs ${DIND} | less"
|
echo " - Inspect DinD logs: docker logs ${DIND} | less"
|
||||||
echo " - Use DinD daemon: docker -H ${DIND_HOST} ps -a"
|
echo " - Use DinD daemon: docker exec ${DIND} docker ps -a"
|
||||||
echo " - Shared tmp vol: docker -H ${DIND_HOST} run --rm -v ${E2E_TMP_VOL}:/tmp alpine:3.20 ls -la /tmp"
|
echo " - Shared tmp vol: docker exec ${DIND} ls -la /tmp"
|
||||||
echo " - DinD docker root: docker -H ${DIND_HOST} run --rm -v ${DIND_VOL}:/var/lib/docker alpine:3.20 ls -la /var/lib/docker/volumes"
|
echo " - DinD docker root: docker exec ${DIND} ls -la /var/lib/docker/volumes"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -124,8 +119,11 @@ cleanup() {
|
|||||||
}
|
}
|
||||||
trap cleanup EXIT INT TERM
|
trap cleanup EXIT INT TERM
|
||||||
|
|
||||||
log "Creating network ${NET} (if missing)"
|
log "(Re)creating network ${NET} with MTU ${DIND_MTU}"
|
||||||
docker network inspect "${NET}" >/dev/null 2>&1 || docker network create "${NET}" >/dev/null
|
docker network rm "${NET}" >/dev/null 2>&1 || true
|
||||||
|
docker network create \
|
||||||
|
--opt com.docker.network.driver.mtu="${DIND_MTU}" \
|
||||||
|
"${NET}" >/dev/null
|
||||||
|
|
||||||
log "Removing old ${DIND} (if any)"
|
log "Removing old ${DIND} (if any)"
|
||||||
docker rm -f "${DIND}" >/dev/null 2>&1 || true
|
docker rm -f "${DIND}" >/dev/null 2>&1 || true
|
||||||
@@ -145,14 +143,14 @@ docker run -d --privileged \
|
|||||||
-e DOCKER_TLS_CERTDIR="" \
|
-e DOCKER_TLS_CERTDIR="" \
|
||||||
-v "${DIND_VOL}:/var/lib/docker" \
|
-v "${DIND_VOL}:/var/lib/docker" \
|
||||||
-v "${E2E_TMP_VOL}:/tmp" \
|
-v "${E2E_TMP_VOL}:/tmp" \
|
||||||
-p 2375:2375 \
|
|
||||||
docker:dind \
|
docker:dind \
|
||||||
--host=tcp://0.0.0.0:2375 \
|
--host=tcp://0.0.0.0:2375 \
|
||||||
--tls=false >/dev/null
|
--tls=false \
|
||||||
|
--mtu="${DIND_MTU}" >/dev/null
|
||||||
|
|
||||||
log "Waiting for DinD to be ready..."
|
log "Waiting for DinD to be ready..."
|
||||||
for i in $(seq 1 "${READY_TIMEOUT_SECONDS}"); do
|
for i in $(seq 1 "${READY_TIMEOUT_SECONDS}"); do
|
||||||
if docker -H "${DIND_HOST}" version >/dev/null 2>&1; then
|
if dind version >/dev/null 2>&1; then
|
||||||
log "DinD is ready."
|
log "DinD is ready."
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
@@ -168,13 +166,13 @@ done
|
|||||||
|
|
||||||
log "Pre-pulling helper images in DinD..."
|
log "Pre-pulling helper images in DinD..."
|
||||||
log " - Pulling: ${RSYNC_IMG}"
|
log " - Pulling: ${RSYNC_IMG}"
|
||||||
docker -H "${DIND_HOST}" pull "${RSYNC_IMG}"
|
dind pull "${RSYNC_IMG}"
|
||||||
|
|
||||||
log "Ensuring alpine exists in DinD (for debug helpers)"
|
log "Ensuring alpine exists in DinD (for debug helpers)"
|
||||||
docker -H "${DIND_HOST}" pull alpine:3.20 >/dev/null
|
dind pull alpine:3.20 >/dev/null
|
||||||
|
|
||||||
log "Loading ${IMG} image into DinD..."
|
log "Loading ${IMG} image into DinD..."
|
||||||
docker save "${IMG}" | docker -H "${DIND_HOST}" load >/dev/null
|
docker save "${IMG}" | dind_stdin load >/dev/null
|
||||||
|
|
||||||
log "Running E2E tests inside DinD"
|
log "Running E2E tests inside DinD"
|
||||||
set +e
|
set +e
|
||||||
@@ -187,7 +185,7 @@ if [ "${DEBUG_SHELL}" = "1" ]; then
|
|||||||
-v "${DIND_VOL}:/var/lib/docker:ro" \
|
-v "${DIND_VOL}:/var/lib/docker:ro" \
|
||||||
-v "${E2E_TMP_VOL}:/tmp" \
|
-v "${E2E_TMP_VOL}:/tmp" \
|
||||||
"${IMG}" \
|
"${IMG}" \
|
||||||
sh -lc '
|
bash -lc '
|
||||||
set -e
|
set -e
|
||||||
if [ ! -f /etc/machine-id ]; then
|
if [ ! -f /etc/machine-id ]; then
|
||||||
mkdir -p /etc
|
mkdir -p /etc
|
||||||
@@ -195,7 +193,7 @@ if [ "${DEBUG_SHELL}" = "1" ]; then
|
|||||||
fi
|
fi
|
||||||
echo ">> DOCKER_HOST=${DOCKER_HOST}"
|
echo ">> DOCKER_HOST=${DOCKER_HOST}"
|
||||||
docker ps -a || true
|
docker ps -a || true
|
||||||
exec sh
|
exec bash
|
||||||
'
|
'
|
||||||
rc=$?
|
rc=$?
|
||||||
else
|
else
|
||||||
@@ -206,7 +204,7 @@ else
|
|||||||
-v "${DIND_VOL}:/var/lib/docker:ro" \
|
-v "${DIND_VOL}:/var/lib/docker:ro" \
|
||||||
-v "${E2E_TMP_VOL}:/tmp" \
|
-v "${E2E_TMP_VOL}:/tmp" \
|
||||||
"${IMG}" \
|
"${IMG}" \
|
||||||
sh -lc '
|
bash -lc '
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
set -x
|
set -x
|
||||||
export PYTHONUNBUFFERED=1
|
export PYTHONUNBUFFERED=1
|
||||||
|
|||||||
@@ -2,10 +2,12 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
|
import sys
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import pandas
|
import pandas
|
||||||
from dirval import create_stamp_file
|
from dirval import create_stamp_file
|
||||||
|
from pandas.errors import EmptyDataError
|
||||||
|
|
||||||
from .cli import parse_args
|
from .cli import parse_args
|
||||||
from .compose import handle_docker_compose_services
|
from .compose import handle_docker_compose_services
|
||||||
@@ -14,8 +16,10 @@ from .docker import (
|
|||||||
change_containers_status,
|
change_containers_status,
|
||||||
containers_using_volume,
|
containers_using_volume,
|
||||||
docker_volume_names,
|
docker_volume_names,
|
||||||
|
filter_stoppable,
|
||||||
get_image_info,
|
get_image_info,
|
||||||
has_image,
|
has_image,
|
||||||
|
is_swarm_task,
|
||||||
)
|
)
|
||||||
from .shell import execute_shell_command
|
from .shell import execute_shell_command
|
||||||
from .volume import backup_volume
|
from .volume import backup_volume
|
||||||
@@ -51,7 +55,9 @@ def is_image_ignored(container: str, images_no_backup_required: list[str]) -> bo
|
|||||||
return any(pat in img for pat in images_no_backup_required)
|
return any(pat in img for pat in images_no_backup_required)
|
||||||
|
|
||||||
|
|
||||||
def volume_is_fully_ignored(containers: list[str], images_no_backup_required: list[str]) -> bool:
|
def volume_is_fully_ignored(
|
||||||
|
containers: list[str], images_no_backup_required: list[str]
|
||||||
|
) -> bool:
|
||||||
"""
|
"""
|
||||||
Skip file backup only if all containers linked to the volume are ignored.
|
Skip file backup only if all containers linked to the volume are ignored.
|
||||||
"""
|
"""
|
||||||
@@ -62,9 +68,13 @@ def volume_is_fully_ignored(containers: list[str], images_no_backup_required: li
|
|||||||
|
|
||||||
def requires_stop(containers: list[str], images_no_stop_required: list[str]) -> bool:
|
def requires_stop(containers: list[str], images_no_stop_required: list[str]) -> bool:
|
||||||
"""
|
"""
|
||||||
Stop is required if ANY container image is NOT in the whitelist patterns.
|
Stop is required if ANY stoppable container image is NOT in the
|
||||||
|
whitelist patterns. Swarm task containers never count: baudolo must
|
||||||
|
not cycle them (see docker.is_swarm_task).
|
||||||
"""
|
"""
|
||||||
for c in containers:
|
for c in containers:
|
||||||
|
if is_swarm_task(c):
|
||||||
|
continue
|
||||||
img = get_image_info(c)
|
img = get_image_info(c)
|
||||||
if not any(pat in img for pat in images_no_stop_required):
|
if not any(pat in img for pat in images_no_stop_required):
|
||||||
return True
|
return True
|
||||||
@@ -77,21 +87,57 @@ def backup_mariadb_or_postgres(
|
|||||||
volume_dir: str,
|
volume_dir: str,
|
||||||
databases_df: "pandas.DataFrame",
|
databases_df: "pandas.DataFrame",
|
||||||
database_containers: list[str],
|
database_containers: list[str],
|
||||||
) -> bool:
|
) -> tuple[bool, bool]:
|
||||||
"""
|
"""
|
||||||
Returns True if the container is a DB container we handled.
|
Returns (is_db_container, dumped_any)
|
||||||
"""
|
"""
|
||||||
for img in ["mariadb", "postgres"]:
|
for img in ["mariadb", "postgres"]:
|
||||||
if has_image(container, img):
|
if has_image(container, img):
|
||||||
backup_database(
|
dumped = backup_database(
|
||||||
container=container,
|
container=container,
|
||||||
volume_dir=volume_dir,
|
volume_dir=volume_dir,
|
||||||
db_type=img,
|
db_type=img,
|
||||||
databases_df=databases_df,
|
databases_df=databases_df,
|
||||||
database_containers=database_containers,
|
database_containers=database_containers,
|
||||||
)
|
)
|
||||||
return True
|
return True, dumped
|
||||||
return False
|
return False, False
|
||||||
|
|
||||||
|
|
||||||
|
def _empty_databases_df() -> "pandas.DataFrame":
|
||||||
|
"""
|
||||||
|
Create an empty DataFrame with the expected schema for databases.csv.
|
||||||
|
|
||||||
|
This allows the backup to continue without DB dumps when the CSV is missing
|
||||||
|
or empty (pandas EmptyDataError).
|
||||||
|
"""
|
||||||
|
return pandas.DataFrame(columns=["instance", "database", "username", "password"])
|
||||||
|
|
||||||
|
|
||||||
|
def _load_databases_df(csv_path: str) -> "pandas.DataFrame":
|
||||||
|
"""
|
||||||
|
Load databases.csv robustly.
|
||||||
|
|
||||||
|
- Missing file -> warn, continue with empty df
|
||||||
|
- Empty file -> warn, continue with empty df
|
||||||
|
- Valid CSV -> return dataframe
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return pandas.read_csv(csv_path, sep=";", keep_default_na=False, dtype=str)
|
||||||
|
except FileNotFoundError:
|
||||||
|
print(
|
||||||
|
f"WARNING: databases.csv not found: {csv_path}. Continuing without database dumps.",
|
||||||
|
file=sys.stderr,
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
return _empty_databases_df()
|
||||||
|
except EmptyDataError:
|
||||||
|
print(
|
||||||
|
f"WARNING: databases.csv exists but is empty: {csv_path}. Continuing without database dumps.",
|
||||||
|
file=sys.stderr,
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
return _empty_databases_df()
|
||||||
|
|
||||||
|
|
||||||
def _backup_dumps_for_volume(
|
def _backup_dumps_for_volume(
|
||||||
@@ -100,21 +146,26 @@ def _backup_dumps_for_volume(
|
|||||||
vol_dir: str,
|
vol_dir: str,
|
||||||
databases_df: "pandas.DataFrame",
|
databases_df: "pandas.DataFrame",
|
||||||
database_containers: list[str],
|
database_containers: list[str],
|
||||||
) -> bool:
|
) -> tuple[bool, bool]:
|
||||||
"""
|
"""
|
||||||
Create DB dumps for any mariadb/postgres containers attached to this volume.
|
Returns (found_db_container, dumped_any)
|
||||||
Returns True if at least one dump was produced.
|
|
||||||
"""
|
"""
|
||||||
|
found_db = False
|
||||||
dumped_any = False
|
dumped_any = False
|
||||||
|
|
||||||
for c in containers:
|
for c in containers:
|
||||||
if backup_mariadb_or_postgres(
|
is_db, dumped = backup_mariadb_or_postgres(
|
||||||
container=c,
|
container=c,
|
||||||
volume_dir=vol_dir,
|
volume_dir=vol_dir,
|
||||||
databases_df=databases_df,
|
databases_df=databases_df,
|
||||||
database_containers=database_containers,
|
database_containers=database_containers,
|
||||||
):
|
)
|
||||||
|
if is_db:
|
||||||
|
found_db = True
|
||||||
|
if dumped:
|
||||||
dumped_any = True
|
dumped_any = True
|
||||||
return dumped_any
|
|
||||||
|
return found_db, dumped_any
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
@@ -126,7 +177,13 @@ def main() -> int:
|
|||||||
versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
|
versions_dir = os.path.join(args.backups_dir, machine_id, args.repo_name)
|
||||||
version_dir = create_version_directory(versions_dir, backup_time)
|
version_dir = create_version_directory(versions_dir, backup_time)
|
||||||
|
|
||||||
databases_df = pandas.read_csv(args.databases_csv, sep=";")
|
# IMPORTANT:
|
||||||
|
# - keep_default_na=False prevents empty fields from turning into NaN
|
||||||
|
# - dtype=str keeps all columns stable for comparisons/validation
|
||||||
|
#
|
||||||
|
# Robust behavior:
|
||||||
|
# - if the file is missing or empty, we continue without DB dumps.
|
||||||
|
databases_df = _load_databases_df(args.databases_csv)
|
||||||
|
|
||||||
print("💾 Start volume backups...", flush=True)
|
print("💾 Start volume backups...", flush=True)
|
||||||
|
|
||||||
@@ -134,50 +191,64 @@ def main() -> int:
|
|||||||
print(f"Start backup routine for volume: {volume_name}", flush=True)
|
print(f"Start backup routine for volume: {volume_name}", flush=True)
|
||||||
containers = containers_using_volume(volume_name)
|
containers = containers_using_volume(volume_name)
|
||||||
|
|
||||||
|
# EARLY SKIP: if all linked containers are ignored, do not create any dirs
|
||||||
|
if volume_is_fully_ignored(containers, args.images_no_backup_required):
|
||||||
|
print(
|
||||||
|
f"Skipping volume '{volume_name}' entirely (all linked containers are ignored).",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
vol_dir = create_volume_directory(version_dir, volume_name)
|
vol_dir = create_volume_directory(version_dir, volume_name)
|
||||||
|
|
||||||
# Old behavior: DB dumps are additional to file backups.
|
found_db, dumped_any = _backup_dumps_for_volume(
|
||||||
_backup_dumps_for_volume(
|
|
||||||
containers=containers,
|
containers=containers,
|
||||||
vol_dir=vol_dir,
|
vol_dir=vol_dir,
|
||||||
databases_df=databases_df,
|
databases_df=databases_df,
|
||||||
database_containers=args.database_containers,
|
database_containers=args.database_containers,
|
||||||
)
|
)
|
||||||
|
|
||||||
# dump-only: skip ALL file rsync backups
|
# dump-only-sql logic:
|
||||||
if args.dump_only:
|
if args.dump_only_sql:
|
||||||
continue
|
if found_db:
|
||||||
|
if not dumped_any:
|
||||||
# skip file backup if all linked containers are ignored
|
print(
|
||||||
if volume_is_fully_ignored(containers, args.images_no_backup_required):
|
f"WARNING: dump-only-sql requested but no DB dump was produced for DB volume '{volume_name}'. "
|
||||||
print(
|
"Falling back to file backup.",
|
||||||
f"Skipping file backup for volume '{volume_name}' (all linked containers are ignored).",
|
flush=True,
|
||||||
flush=True,
|
)
|
||||||
)
|
# fall through to file backup below
|
||||||
continue
|
else:
|
||||||
|
# DB volume successfully dumped -> skip file backup
|
||||||
|
continue
|
||||||
|
# Non-DB volume -> always do file backup (fall through)
|
||||||
|
|
||||||
if args.everything:
|
if args.everything:
|
||||||
# "everything": always do pre-rsync, then stop + rsync again
|
# "everything": always do pre-rsync, then stop + rsync again
|
||||||
|
stoppable = filter_stoppable(containers)
|
||||||
backup_volume(versions_dir, volume_name, vol_dir)
|
backup_volume(versions_dir, volume_name, vol_dir)
|
||||||
change_containers_status(containers, "stop")
|
change_containers_status(stoppable, "stop")
|
||||||
backup_volume(versions_dir, volume_name, vol_dir)
|
backup_volume(versions_dir, volume_name, vol_dir)
|
||||||
if not args.shutdown:
|
if not args.shutdown:
|
||||||
change_containers_status(containers, "start")
|
change_containers_status(stoppable, "start")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# default: rsync, and if needed stop + rsync
|
# default: rsync, and if needed stop + rsync
|
||||||
backup_volume(versions_dir, volume_name, vol_dir)
|
backup_volume(versions_dir, volume_name, vol_dir)
|
||||||
if requires_stop(containers, args.images_no_stop_required):
|
if requires_stop(containers, args.images_no_stop_required):
|
||||||
change_containers_status(containers, "stop")
|
stoppable = filter_stoppable(containers)
|
||||||
|
change_containers_status(stoppable, "stop")
|
||||||
backup_volume(versions_dir, volume_name, vol_dir)
|
backup_volume(versions_dir, volume_name, vol_dir)
|
||||||
if not args.shutdown:
|
if not args.shutdown:
|
||||||
change_containers_status(containers, "start")
|
change_containers_status(stoppable, "start")
|
||||||
|
|
||||||
# Stamp the backup version directory using dirval (python lib)
|
# Stamp the backup version directory using dirval (python lib)
|
||||||
stamp_directory(version_dir)
|
stamp_directory(version_dir)
|
||||||
print("Finished volume backups.", flush=True)
|
print("Finished volume backups.", flush=True)
|
||||||
|
|
||||||
print("Handling Docker Compose services...", flush=True)
|
print("Handling Docker Compose services...", flush=True)
|
||||||
handle_docker_compose_services(args.compose_dir, args.docker_compose_hard_restart_required)
|
handle_docker_compose_services(
|
||||||
|
args.compose_dir, args.docker_compose_hard_restart_required
|
||||||
|
)
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|||||||
@@ -2,22 +2,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
def _default_repo_name() -> str:
|
|
||||||
"""
|
|
||||||
Derive the repository name from the folder that contains `src/`.
|
|
||||||
|
|
||||||
Expected layout:
|
|
||||||
<repo-root>/src/baudolo/backup/cli.py
|
|
||||||
|
|
||||||
=> parents[0]=backup, [1]=baudolo, [2]=src, [3]=repo-root
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
return Path(__file__).resolve().parents[3].name
|
|
||||||
except Exception:
|
|
||||||
return "backup-docker-to-local"
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
def parse_args() -> argparse.Namespace:
|
||||||
@@ -41,7 +25,7 @@ def parse_args() -> argparse.Namespace:
|
|||||||
|
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--repo-name",
|
"--repo-name",
|
||||||
default=_default_repo_name(),
|
default="backup-docker-to-local",
|
||||||
help="Backup repo folder name under <backups-dir>/<machine-id>/ (default: git repo folder name)",
|
help="Backup repo folder name under <backups-dir>/<machine-id>/ (default: git repo folder name)",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
@@ -51,8 +35,8 @@ def parse_args() -> argparse.Namespace:
|
|||||||
)
|
)
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
"--backups-dir",
|
"--backups-dir",
|
||||||
default="/Backups",
|
default="/var/lib/backup/",
|
||||||
help="Backup root directory (default: /Backups)",
|
help="Backup root directory (default: /var/lib/backup/)",
|
||||||
)
|
)
|
||||||
|
|
||||||
p.add_argument(
|
p.add_argument(
|
||||||
@@ -84,10 +68,15 @@ def parse_args() -> argparse.Namespace:
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="Do not restart containers after backup",
|
help="Do not restart containers after backup",
|
||||||
)
|
)
|
||||||
p.add_argument(
|
|
||||||
"--dump-only",
|
|
||||||
action="store_true",
|
|
||||||
help="Only create DB dumps (skip ALL file rsync backups)",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
p.add_argument(
|
||||||
|
"--dump-only-sql",
|
||||||
|
action="store_true",
|
||||||
|
help=(
|
||||||
|
"Create database dumps only for DB volumes. "
|
||||||
|
"File backups are skipped for DB volumes if a dump succeeds, "
|
||||||
|
"but non-DB volumes are still backed up. "
|
||||||
|
"If a DB dump cannot be produced, baudolo falls back to a file backup."
|
||||||
|
),
|
||||||
|
)
|
||||||
return p.parse_args()
|
return p.parse_args()
|
||||||
|
|||||||
@@ -1,27 +1,95 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
|
|
||||||
|
def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
|
||||||
|
"""
|
||||||
|
Build the compose command for this project directory.
|
||||||
|
|
||||||
|
Policy:
|
||||||
|
- If `compose` wrapper exists (Infinito.Nexus): use it and delegate ALL logic to it.
|
||||||
|
- Else: use plain `docker compose` with --chdir.
|
||||||
|
- NO custom compose file/env detection in this project.
|
||||||
|
"""
|
||||||
|
pdir = Path(project_dir).resolve()
|
||||||
|
|
||||||
|
wrapper = shutil.which("compose")
|
||||||
|
if wrapper:
|
||||||
|
# "--" ensures wrapper stops parsing its own args.
|
||||||
|
return [wrapper, "--chdir", str(pdir), "--", *passthrough]
|
||||||
|
|
||||||
|
docker = shutil.which("docker")
|
||||||
|
if docker:
|
||||||
|
return [docker, "compose", "--chdir", str(pdir), *passthrough]
|
||||||
|
|
||||||
|
raise RuntimeError("Neither 'compose' nor 'docker' found in PATH")
|
||||||
|
|
||||||
|
|
||||||
|
def _find_compose_file(project_dir: str) -> Optional[Path]:
|
||||||
|
"""
|
||||||
|
Detect a compose file in `project_dir` (case-insensitive).
|
||||||
|
|
||||||
|
Supported names:
|
||||||
|
- compose.yml / compose.yaml
|
||||||
|
- docker-compose.yml / docker-compose.yaml
|
||||||
|
"""
|
||||||
|
pdir = Path(project_dir)
|
||||||
|
if not pdir.is_dir():
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Map lowercase filename -> actual Path (preserves original casing)
|
||||||
|
by_lower = {p.name.lower(): p for p in pdir.iterdir() if p.is_file()}
|
||||||
|
|
||||||
|
# Preferred order (policy decision)
|
||||||
|
candidates = [
|
||||||
|
"docker-compose.yml",
|
||||||
|
"docker-compose.yaml",
|
||||||
|
"compose.yml",
|
||||||
|
"compose.yaml",
|
||||||
|
]
|
||||||
|
|
||||||
|
for name in candidates:
|
||||||
|
found = by_lower.get(name)
|
||||||
|
if found is not None:
|
||||||
|
return found
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def hard_restart_docker_services(dir_path: str) -> None:
|
def hard_restart_docker_services(dir_path: str) -> None:
|
||||||
print(f"Hard restart docker-compose services in: {dir_path}", flush=True)
|
print(f"Hard restart compose services in: {dir_path}", flush=True)
|
||||||
subprocess.run(["docker-compose", "down"], cwd=dir_path, check=True)
|
|
||||||
subprocess.run(["docker-compose", "up", "-d"], cwd=dir_path, check=True)
|
down_cmd = _build_compose_cmd(dir_path, ["down"])
|
||||||
|
up_cmd = _build_compose_cmd(dir_path, ["up", "-d"])
|
||||||
|
|
||||||
|
print(">>> " + " ".join(down_cmd), flush=True)
|
||||||
|
subprocess.run(down_cmd, check=True)
|
||||||
|
|
||||||
|
print(">>> " + " ".join(up_cmd), flush=True)
|
||||||
|
subprocess.run(up_cmd, check=True)
|
||||||
|
|
||||||
|
|
||||||
def handle_docker_compose_services(parent_directory: str, hard_restart_required: list[str]) -> None:
|
def handle_docker_compose_services(
|
||||||
|
parent_directory: str,
|
||||||
|
hard_restart_required: list[str],
|
||||||
|
) -> None:
|
||||||
for entry in os.scandir(parent_directory):
|
for entry in os.scandir(parent_directory):
|
||||||
if not entry.is_dir():
|
if not entry.is_dir():
|
||||||
continue
|
continue
|
||||||
|
|
||||||
dir_path = entry.path
|
dir_path = entry.path
|
||||||
name = os.path.basename(dir_path)
|
name = os.path.basename(dir_path)
|
||||||
compose_file = os.path.join(dir_path, "docker-compose.yml")
|
|
||||||
|
|
||||||
print(f"Checking directory: {dir_path}", flush=True)
|
print(f"Checking directory: {dir_path}", flush=True)
|
||||||
if not os.path.isfile(compose_file):
|
|
||||||
print("No docker-compose.yml found. Skipping.", flush=True)
|
compose_file = _find_compose_file(dir_path)
|
||||||
|
if compose_file is None:
|
||||||
|
print("No supported compose file found. Skipping.", flush=True)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if name in hard_restart_required:
|
if name in hard_restart_required:
|
||||||
|
|||||||
@@ -3,24 +3,66 @@ from __future__ import annotations
|
|||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
import re
|
import re
|
||||||
|
import logging
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
import pandas
|
import pandas
|
||||||
|
|
||||||
from .shell import BackupException, execute_shell_command
|
from .shell import BackupException, execute_shell_command
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def get_instance(container: str, database_containers: list[str]) -> str:
|
def get_instance(container: str, database_containers: list[str]) -> str:
|
||||||
|
"""
|
||||||
|
Derive a stable instance name from the container name.
|
||||||
|
"""
|
||||||
if container in database_containers:
|
if container in database_containers:
|
||||||
return container
|
return container
|
||||||
return re.split(r"(_|-)(database|db|postgres)", container)[0]
|
return re.split(r"(_|-)(database|db|postgres)", container)[0]
|
||||||
|
|
||||||
|
|
||||||
def fallback_pg_dumpall(container: str, username: str, password: str, out_file: str) -> None:
|
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
|
||||||
|
"""
|
||||||
|
Enforce explicit database semantics:
|
||||||
|
|
||||||
|
- "*" => dump ALL databases (cluster dump for Postgres)
|
||||||
|
- "<name>" => dump exactly this database
|
||||||
|
- "" => invalid configuration (would previously result in NaN / nan.backup.sql)
|
||||||
|
"""
|
||||||
|
v = (value or "").strip()
|
||||||
|
if v == "":
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid databases.csv entry for instance '{instance}': "
|
||||||
|
"column 'database' must be '*' or a concrete database name (not empty)."
|
||||||
|
)
|
||||||
|
return v
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_write_cmd(cmd: str, out_file: str) -> None:
|
||||||
|
"""
|
||||||
|
Write dump output atomically:
|
||||||
|
- write to <file>.tmp
|
||||||
|
- rename to <file> only on success
|
||||||
|
|
||||||
|
This prevents empty or partial dump files from being treated as valid backups.
|
||||||
|
"""
|
||||||
|
tmp = f"{out_file}.tmp"
|
||||||
|
execute_shell_command(f"{cmd} > {tmp}")
|
||||||
|
execute_shell_command(f"mv {tmp} {out_file}")
|
||||||
|
|
||||||
|
|
||||||
|
def fallback_pg_dumpall(
|
||||||
|
container: str, username: str, password: str, out_file: str
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Perform a full Postgres cluster dump using pg_dumpall.
|
||||||
|
"""
|
||||||
cmd = (
|
cmd = (
|
||||||
f"PGPASSWORD={password} docker exec -i {container} "
|
f"PGPASSWORD={password} docker exec -i {container} "
|
||||||
f"pg_dumpall -U {username} -h localhost > {out_file}"
|
f"pg_dumpall -U {username} -h localhost"
|
||||||
)
|
)
|
||||||
execute_shell_command(cmd)
|
_atomic_write_cmd(cmd, out_file)
|
||||||
|
|
||||||
|
|
||||||
def backup_database(
|
def backup_database(
|
||||||
@@ -30,44 +72,76 @@ def backup_database(
|
|||||||
db_type: str,
|
db_type: str,
|
||||||
databases_df: "pandas.DataFrame",
|
databases_df: "pandas.DataFrame",
|
||||||
database_containers: list[str],
|
database_containers: list[str],
|
||||||
) -> None:
|
) -> bool:
|
||||||
|
"""
|
||||||
|
Backup databases for a given DB container.
|
||||||
|
|
||||||
|
Returns True if at least one dump was produced.
|
||||||
|
"""
|
||||||
instance_name = get_instance(container, database_containers)
|
instance_name = get_instance(container, database_containers)
|
||||||
entries = databases_df.loc[databases_df["instance"] == instance_name]
|
|
||||||
|
entries = databases_df[databases_df["instance"] == instance_name]
|
||||||
if entries.empty:
|
if entries.empty:
|
||||||
raise BackupException(f"No entry found for instance '{instance_name}'")
|
log.debug("No database entries for instance '%s'", instance_name)
|
||||||
|
return False
|
||||||
|
|
||||||
out_dir = os.path.join(volume_dir, "sql")
|
out_dir = os.path.join(volume_dir, "sql")
|
||||||
pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
|
pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
for row in entries.iloc:
|
produced = False
|
||||||
db_name = row["database"]
|
|
||||||
user = row["username"]
|
|
||||||
password = row["password"]
|
|
||||||
|
|
||||||
|
for row in entries.itertuples(index=False):
|
||||||
|
raw_db = getattr(row, "database", "")
|
||||||
|
user = (getattr(row, "username", "") or "").strip()
|
||||||
|
password = (getattr(row, "password", "") or "").strip()
|
||||||
|
|
||||||
|
db_value = _validate_database_value(raw_db, instance=instance_name)
|
||||||
|
|
||||||
|
# Explicit: dump ALL databases
|
||||||
|
if db_value == "*":
|
||||||
|
if db_type != "postgres":
|
||||||
|
raise ValueError(
|
||||||
|
f"databases.csv entry for instance '{instance_name}': "
|
||||||
|
"'*' is currently only supported for Postgres."
|
||||||
|
)
|
||||||
|
|
||||||
|
cluster_file = os.path.join(out_dir, f"{instance_name}.cluster.backup.sql")
|
||||||
|
fallback_pg_dumpall(container, user, password, cluster_file)
|
||||||
|
produced = True
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Concrete database dump
|
||||||
|
db_name = db_value
|
||||||
dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
|
dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
|
||||||
|
|
||||||
if db_type == "mariadb":
|
if db_type == "mariadb":
|
||||||
|
# Force TCP so auth matches '<user>'@'%' instead of socket -> 'localhost'.
|
||||||
cmd = (
|
cmd = (
|
||||||
f"docker exec {container} /usr/bin/mariadb-dump "
|
f"docker exec {container} /usr/bin/mariadb-dump "
|
||||||
f"-u {user} -p{password} {db_name} > {dump_file}"
|
f"-h 127.0.0.1 --protocol=tcp "
|
||||||
|
f"-u {user} -p{password} {db_name}"
|
||||||
)
|
)
|
||||||
execute_shell_command(cmd)
|
_atomic_write_cmd(cmd, dump_file)
|
||||||
|
produced = True
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if db_type == "postgres":
|
if db_type == "postgres":
|
||||||
cluster_file = os.path.join(out_dir, f"{instance_name}.cluster.backup.sql")
|
|
||||||
if not db_name:
|
|
||||||
fallback_pg_dumpall(container, user, password, cluster_file)
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
cmd = (
|
cmd = (
|
||||||
f"PGPASSWORD={password} docker exec -i {container} "
|
f"PGPASSWORD={password} docker exec -i {container} "
|
||||||
f"pg_dump -U {user} -d {db_name} -h localhost > {dump_file}"
|
f"pg_dump -U {user} -d {db_name} -h localhost "
|
||||||
|
f"--no-owner --no-privileges"
|
||||||
)
|
)
|
||||||
execute_shell_command(cmd)
|
_atomic_write_cmd(cmd, dump_file)
|
||||||
|
produced = True
|
||||||
except BackupException as e:
|
except BackupException as e:
|
||||||
print(f"pg_dump failed: {e}", flush=True)
|
# Explicit DB dump failed -> hard error
|
||||||
print(f"Falling back to pg_dumpall for instance '{instance_name}'", flush=True)
|
raise BackupException(
|
||||||
fallback_pg_dumpall(container, user, password, cluster_file)
|
f"Postgres dump failed for instance '{instance_name}', "
|
||||||
|
f"database '{db_name}'. This database was explicitly configured "
|
||||||
|
"and therefore must succeed.\n"
|
||||||
|
f"{e}"
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
return produced
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from .shell import execute_shell_command
|
from .shell import BackupException, execute_shell_command
|
||||||
|
|
||||||
|
|
||||||
def get_image_info(container: str) -> str:
|
def get_image_info(container: str) -> str:
|
||||||
@@ -24,6 +24,43 @@ def containers_using_volume(volume_name: str) -> list[str]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_swarm_task(container: str) -> bool:
|
||||||
|
"""Swarm-managed task containers must never be stopped or started
|
||||||
|
manually: the orchestrator replaces the stopped task and a later
|
||||||
|
`docker start` fails on the detached overlay network. A container that
|
||||||
|
vanished between listing and inspect (--rm one-shots, task-history GC)
|
||||||
|
counts as not stoppable instead of aborting the whole backup run; if the
|
||||||
|
container still exists the inspect failure re-raises, so a broken daemon
|
||||||
|
keeps failing the run loudly instead of silently skipping the stop."""
|
||||||
|
try:
|
||||||
|
out = execute_shell_command(
|
||||||
|
"docker inspect --format "
|
||||||
|
f"'{{{{index .Config.Labels \"com.docker.swarm.task.id\"}}}}' {container}"
|
||||||
|
)
|
||||||
|
except BackupException:
|
||||||
|
still_listed = execute_shell_command(
|
||||||
|
f"docker ps -a --filter name=^{container}$ --format '{{{{.Names}}}}'"
|
||||||
|
)
|
||||||
|
if still_listed and still_listed[0].strip():
|
||||||
|
raise
|
||||||
|
return True
|
||||||
|
return bool(out and out[0].strip())
|
||||||
|
|
||||||
|
|
||||||
|
def filter_stoppable(containers: list[str]) -> list[str]:
|
||||||
|
"""Containers baudolo may stop/start itself (everything but swarm tasks)."""
|
||||||
|
stoppable = []
|
||||||
|
for container in containers:
|
||||||
|
if is_swarm_task(container):
|
||||||
|
print(
|
||||||
|
f"Skipping stop/start for swarm task container '{container}'.",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
stoppable.append(container)
|
||||||
|
return stoppable
|
||||||
|
|
||||||
|
|
||||||
def change_containers_status(containers: list[str], status: str) -> None:
|
def change_containers_status(containers: list[str], status: str) -> None:
|
||||||
"""Stop or start a list of containers."""
|
"""Stop or start a list of containers."""
|
||||||
if not containers:
|
if not containers:
|
||||||
@@ -37,7 +74,9 @@ def change_containers_status(containers: list[str], status: str) -> None:
|
|||||||
def docker_volume_exists(volume: str) -> bool:
|
def docker_volume_exists(volume: str) -> bool:
|
||||||
# Avoid throwing exceptions for exists checks.
|
# Avoid throwing exceptions for exists checks.
|
||||||
try:
|
try:
|
||||||
execute_shell_command(f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK")
|
execute_shell_command(
|
||||||
|
f"docker volume inspect {volume} >/dev/null 2>&1 && echo OK"
|
||||||
|
)
|
||||||
return True
|
return True
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
|
|||||||
@@ -13,7 +13,9 @@ def get_storage_path(volume_name: str) -> str:
|
|||||||
return f"{path}/"
|
return f"{path}/"
|
||||||
|
|
||||||
|
|
||||||
def get_last_backup_dir(versions_dir: str, volume_name: str, current_backup_dir: str) -> str | None:
|
def get_last_backup_dir(
|
||||||
|
versions_dir: str, volume_name: str, current_backup_dir: str
|
||||||
|
) -> str | None:
|
||||||
versions = sorted(os.listdir(versions_dir), reverse=True)
|
versions = sorted(os.listdir(versions_dir), reverse=True)
|
||||||
for version in versions:
|
for version in versions:
|
||||||
candidate = os.path.join(versions_dir, version, volume_name, "files", "")
|
candidate = os.path.join(versions_dir, version, volume_name, "files", "")
|
||||||
@@ -37,6 +39,8 @@ def backup_volume(versions_dir: str, volume_name: str, volume_dir: str) -> None:
|
|||||||
execute_shell_command(cmd)
|
execute_shell_command(cmd)
|
||||||
except BackupException as e:
|
except BackupException as e:
|
||||||
if "file has vanished" in str(e):
|
if "file has vanished" in str(e):
|
||||||
print("Warning: Some files vanished before transfer. Continuing.", flush=True)
|
print(
|
||||||
|
"Warning: Some files vanished before transfer. Continuing.", flush=True
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
__all__ = ["main"]
|
__all__ = ["main"]
|
||||||
|
|||||||
@@ -66,7 +66,9 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# mariadb
|
# mariadb
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
p_mdb = sub.add_parser("mariadb", help="Restore a single MariaDB/MySQL-compatible dump")
|
p_mdb = sub.add_parser(
|
||||||
|
"mariadb", help="Restore a single MariaDB/MySQL-compatible dump"
|
||||||
|
)
|
||||||
_add_common_backup_args(p_mdb)
|
_add_common_backup_args(p_mdb)
|
||||||
p_mdb.add_argument("--container", required=True)
|
p_mdb.add_argument("--container", required=True)
|
||||||
p_mdb.add_argument("--db-name", required=True)
|
p_mdb.add_argument("--db-name", required=True)
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
"""Database restore handlers (Postgres, MariaDB/MySQL)."""
|
"""Database restore handlers (Postgres, MariaDB/MySQL)."""
|
||||||
|
|||||||
@@ -23,7 +23,9 @@ exit 42
|
|||||||
raise RuntimeError("empty client detection output")
|
raise RuntimeError("empty client detection output")
|
||||||
return out
|
return out
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print("ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr)
|
print(
|
||||||
|
"ERROR: neither 'mariadb' nor 'mysql' found in container.", file=sys.stderr
|
||||||
|
)
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
|
|
||||||
@@ -45,11 +47,6 @@ def restore_mariadb_sql(
|
|||||||
# IMPORTANT:
|
# IMPORTANT:
|
||||||
# Do NOT hardcode 'mysql' here. Use the detected client.
|
# Do NOT hardcode 'mysql' here. Use the detected client.
|
||||||
# MariaDB 11 images may not contain the mysql binary at all.
|
# MariaDB 11 images may not contain the mysql binary at all.
|
||||||
docker_exec(
|
|
||||||
container,
|
|
||||||
[client, "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=0;"],
|
|
||||||
)
|
|
||||||
|
|
||||||
result = docker_exec(
|
result = docker_exec(
|
||||||
container,
|
container,
|
||||||
[
|
[
|
||||||
@@ -65,7 +62,16 @@ def restore_mariadb_sql(
|
|||||||
)
|
)
|
||||||
tables = result.stdout.decode().split()
|
tables = result.stdout.decode().split()
|
||||||
|
|
||||||
for tbl in tables:
|
if tables:
|
||||||
|
# SET FOREIGN_KEY_CHECKS is session-scoped, so it must share one
|
||||||
|
# client session with the DROPs or FK constraints still fire.
|
||||||
|
drop_sql = (
|
||||||
|
"SET FOREIGN_KEY_CHECKS=0; "
|
||||||
|
+ " ".join(
|
||||||
|
f"DROP TABLE IF EXISTS `{db_name}`.`{tbl}`;" for tbl in tables
|
||||||
|
)
|
||||||
|
+ " SET FOREIGN_KEY_CHECKS=1;"
|
||||||
|
)
|
||||||
docker_exec(
|
docker_exec(
|
||||||
container,
|
container,
|
||||||
[
|
[
|
||||||
@@ -74,16 +80,13 @@ def restore_mariadb_sql(
|
|||||||
user,
|
user,
|
||||||
f"--password={password}",
|
f"--password={password}",
|
||||||
"-e",
|
"-e",
|
||||||
f"DROP TABLE IF EXISTS `{db_name}`.`{tbl}`;",
|
drop_sql,
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
with open(sql_path, "rb") as f:
|
||||||
docker_exec(
|
docker_exec(
|
||||||
container,
|
container, [client, "-u", user, f"--password={password}", db_name], stdin=f
|
||||||
[client, "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=1;"],
|
|
||||||
)
|
)
|
||||||
|
|
||||||
with open(sql_path, "rb") as f:
|
|
||||||
docker_exec(container, [client, "-u", user, f"--password={password}", db_name], stdin=f)
|
|
||||||
|
|
||||||
print(f"MariaDB/MySQL restore complete for db '{db_name}'.")
|
print(f"MariaDB/MySQL restore complete for db '{db_name}'.")
|
||||||
|
|||||||
@@ -1,9 +1,41 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import tempfile
|
||||||
|
from collections.abc import Iterable, Iterator
|
||||||
|
|
||||||
from ..run import docker_exec
|
from ..run import docker_exec
|
||||||
|
|
||||||
|
_SUPERUSER_ONLY_PREFIXES = (b"COMMENT ON EXTENSION", b"ALTER DEFAULT PRIVILEGES")
|
||||||
|
|
||||||
|
|
||||||
|
def filter_superuser_only_lines(lines: Iterable[bytes]) -> Iterator[bytes]:
|
||||||
|
"""Drop superuser-only statements an app-level psql replay cannot run.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
lines: dump lines including their trailing newlines.
|
||||||
|
|
||||||
|
Yields:
|
||||||
|
Every line except top-level statements starting with a superuser-only
|
||||||
|
prefix. Lines inside COPY ... FROM stdin data blocks are passed
|
||||||
|
through untouched: a data row may legally start with the same bytes,
|
||||||
|
and dropping it would silently corrupt the restored table.
|
||||||
|
"""
|
||||||
|
in_copy = False
|
||||||
|
for line in lines:
|
||||||
|
if in_copy:
|
||||||
|
yield line
|
||||||
|
if line.rstrip(b"\r\n") == b"\\.":
|
||||||
|
in_copy = False
|
||||||
|
continue
|
||||||
|
if line.startswith(b"COPY ") and line.rstrip(b"\r\n").endswith(b"FROM stdin;"):
|
||||||
|
in_copy = True
|
||||||
|
yield line
|
||||||
|
continue
|
||||||
|
if line.startswith(_SUPERUSER_ONLY_PREFIXES):
|
||||||
|
continue
|
||||||
|
yield line
|
||||||
|
|
||||||
|
|
||||||
def restore_postgres_sql(
|
def restore_postgres_sql(
|
||||||
*,
|
*,
|
||||||
@@ -21,17 +53,48 @@ def restore_postgres_sql(
|
|||||||
docker_env = {"PGPASSWORD": password}
|
docker_env = {"PGPASSWORD": password}
|
||||||
|
|
||||||
if empty:
|
if empty:
|
||||||
|
# Owner-filtered: extension members (pg_trgm's set_limit) are superuser-owned; IF EXISTS absorbs CASCADE fallout.
|
||||||
drop_sql = r"""
|
drop_sql = r"""
|
||||||
DO $$ DECLARE r RECORD;
|
DO $$ DECLARE r RECORD;
|
||||||
BEGIN
|
BEGIN
|
||||||
FOR r IN (
|
FOR r IN (
|
||||||
SELECT table_name AS name, 'TABLE' AS type FROM information_schema.tables WHERE table_schema='public'
|
SELECT c.relname AS name,
|
||||||
|
CASE c.relkind
|
||||||
|
WHEN 'v' THEN 'VIEW'
|
||||||
|
WHEN 'm' THEN 'MATERIALIZED VIEW'
|
||||||
|
WHEN 'f' THEN 'FOREIGN TABLE'
|
||||||
|
ELSE 'TABLE'
|
||||||
|
END AS type
|
||||||
|
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
|
WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p', 'v', 'm', 'f')
|
||||||
|
AND pg_get_userbyid(c.relowner) = current_user
|
||||||
UNION ALL
|
UNION ALL
|
||||||
SELECT routine_name AS name, 'FUNCTION' AS type FROM information_schema.routines WHERE specific_schema='public'
|
SELECT p.proname AS name,
|
||||||
|
CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END AS type
|
||||||
|
FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
|
||||||
|
WHERE n.nspname = 'public' AND p.prokind IN ('f', 'p', 'w')
|
||||||
|
AND pg_get_userbyid(p.proowner) = current_user
|
||||||
UNION ALL
|
UNION ALL
|
||||||
SELECT sequence_name AS name, 'SEQUENCE' AS type FROM information_schema.sequences WHERE sequence_schema='public'
|
SELECT c.relname AS name, 'SEQUENCE' AS type
|
||||||
|
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
|
WHERE n.nspname = 'public' AND c.relkind = 'S'
|
||||||
|
AND pg_get_userbyid(c.relowner) = current_user
|
||||||
|
UNION ALL
|
||||||
|
SELECT t.typname AS name, 'TYPE' AS type
|
||||||
|
FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
|
||||||
|
WHERE n.nspname = 'public'
|
||||||
|
AND pg_get_userbyid(t.typowner) = current_user
|
||||||
|
AND (t.typtype IN ('e', 'd')
|
||||||
|
OR (t.typtype = 'c' AND EXISTS (
|
||||||
|
SELECT 1 FROM pg_class c2
|
||||||
|
WHERE c2.oid = t.typrelid AND c2.relkind = 'c')))
|
||||||
|
UNION ALL
|
||||||
|
SELECT col.collname AS name, 'COLLATION' AS type
|
||||||
|
FROM pg_collation col JOIN pg_namespace n ON n.oid = col.collnamespace
|
||||||
|
WHERE n.nspname = 'public'
|
||||||
|
AND pg_get_userbyid(col.collowner) = current_user
|
||||||
) LOOP
|
) LOOP
|
||||||
EXECUTE format('DROP %s public.%I CASCADE', r.type, r.name);
|
EXECUTE format('DROP %s IF EXISTS public.%I CASCADE', r.type, r.name);
|
||||||
END LOOP;
|
END LOOP;
|
||||||
END $$;
|
END $$;
|
||||||
"""
|
"""
|
||||||
@@ -42,11 +105,17 @@ END $$;
|
|||||||
docker_env=docker_env,
|
docker_env=docker_env,
|
||||||
)
|
)
|
||||||
|
|
||||||
with open(sql_path, "rb") as f:
|
# Filter into a spooled temp file instead of building the whole dump in
|
||||||
|
# memory: production dumps reach many GB and the previous read/splitlines/
|
||||||
|
# join needed roughly three times the dump size in RSS.
|
||||||
|
with open(sql_path, "rb") as src, tempfile.TemporaryFile() as filtered:
|
||||||
|
for line in filter_superuser_only_lines(src):
|
||||||
|
filtered.write(line)
|
||||||
|
filtered.seek(0)
|
||||||
docker_exec(
|
docker_exec(
|
||||||
container,
|
container,
|
||||||
["psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", db_name],
|
["psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", db_name],
|
||||||
stdin=f,
|
stdin=filtered,
|
||||||
docker_env=docker_env,
|
docker_env=docker_env,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,9 @@ import sys
|
|||||||
from .run import run, docker_volume_exists
|
from .run import run, docker_volume_exists
|
||||||
|
|
||||||
|
|
||||||
def restore_volume_files(volume_name: str, backup_files_dir: str, *, rsync_image: str) -> int:
|
def restore_volume_files(
|
||||||
|
volume_name: str, backup_files_dir: str, *, rsync_image: str
|
||||||
|
) -> int:
|
||||||
if not os.path.isdir(backup_files_dir):
|
if not os.path.isdir(backup_files_dir):
|
||||||
print(f"ERROR: backup files dir not found: {backup_files_dir}", file=sys.stderr)
|
print(f"ERROR: backup files dir not found: {backup_files_dir}", file=sys.stderr)
|
||||||
return 2
|
return 2
|
||||||
|
|||||||
@@ -1,50 +1,117 @@
|
|||||||
import pandas as pd
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import pandas as pd
|
||||||
|
from typing import Optional
|
||||||
|
from pandas.errors import EmptyDataError
|
||||||
|
|
||||||
def check_and_add_entry(file_path, instance, database, username, password):
|
DB_NAME_RE = re.compile(r"^[a-zA-Z0-9_][a-zA-Z0-9_-]*$")
|
||||||
# Check if the file exists and is not empty
|
|
||||||
if os.path.exists(file_path) and os.path.getsize(file_path) > 0:
|
|
||||||
# Read the existing CSV file with header
|
def _validate_database_value(value: Optional[str], *, instance: str) -> str:
|
||||||
df = pd.read_csv(file_path, sep=';')
|
v = (value or "").strip()
|
||||||
|
if v == "":
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid databases.csv entry for instance '{instance}': "
|
||||||
|
"column 'database' must be '*' or a concrete database name (not empty)."
|
||||||
|
)
|
||||||
|
if v == "*":
|
||||||
|
return "*"
|
||||||
|
if v.lower() == "nan":
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid databases.csv entry for instance '{instance}': database must not be 'nan'."
|
||||||
|
)
|
||||||
|
if not DB_NAME_RE.match(v):
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid databases.csv entry for instance '{instance}': "
|
||||||
|
f"invalid database name '{v}'. Allowed: letters, numbers, '_' and '-'."
|
||||||
|
)
|
||||||
|
return v
|
||||||
|
|
||||||
|
|
||||||
|
def _empty_df() -> pd.DataFrame:
|
||||||
|
return pd.DataFrame(columns=["instance", "database", "username", "password"])
|
||||||
|
|
||||||
|
|
||||||
|
def check_and_add_entry(
|
||||||
|
file_path: str,
|
||||||
|
instance: str,
|
||||||
|
database: Optional[str],
|
||||||
|
username: str,
|
||||||
|
password: str,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Add or update an entry in databases.csv.
|
||||||
|
|
||||||
|
The function enforces strict validation:
|
||||||
|
- database MUST be set
|
||||||
|
- database MUST be '*' or a valid database name
|
||||||
|
"""
|
||||||
|
database = _validate_database_value(database, instance=instance)
|
||||||
|
|
||||||
|
if os.path.exists(file_path):
|
||||||
|
try:
|
||||||
|
df = pd.read_csv(
|
||||||
|
file_path,
|
||||||
|
sep=";",
|
||||||
|
dtype=str,
|
||||||
|
keep_default_na=False,
|
||||||
|
)
|
||||||
|
except EmptyDataError:
|
||||||
|
print(
|
||||||
|
f"WARNING: databases.csv exists but is empty: {file_path}. Creating header columns.",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
df = _empty_df()
|
||||||
else:
|
else:
|
||||||
# Create a new DataFrame with columns if file does not exist
|
df = _empty_df()
|
||||||
df = pd.DataFrame(columns=['instance', 'database', 'username', 'password'])
|
mask = (df["instance"] == instance) & (df["database"] == database)
|
||||||
|
|
||||||
# Check if the entry exists and remove it
|
if mask.any():
|
||||||
mask = (
|
print("Updating existing entry.")
|
||||||
(df['instance'] == instance) &
|
df.loc[mask, ["username", "password"]] = [username, password]
|
||||||
((df['database'] == database) |
|
|
||||||
(((df['database'].isna()) | (df['database'] == '')) & (database == ''))) &
|
|
||||||
(df['username'] == username)
|
|
||||||
)
|
|
||||||
|
|
||||||
if not df[mask].empty:
|
|
||||||
print("Replacing existing entry.")
|
|
||||||
df = df[~mask]
|
|
||||||
else:
|
else:
|
||||||
print("Adding new entry.")
|
print("Adding new entry.")
|
||||||
|
new_entry = pd.DataFrame(
|
||||||
|
[[instance, database, username, password]],
|
||||||
|
columns=["instance", "database", "username", "password"],
|
||||||
|
)
|
||||||
|
df = pd.concat([df, new_entry], ignore_index=True)
|
||||||
|
|
||||||
# Create a new DataFrame for the new entry
|
df.to_csv(file_path, sep=";", index=False)
|
||||||
new_entry = pd.DataFrame([{'instance': instance, 'database': database, 'username': username, 'password': password}])
|
|
||||||
|
|
||||||
# Add (or replace) the entry using concat
|
|
||||||
df = pd.concat([df, new_entry], ignore_index=True)
|
|
||||||
|
|
||||||
# Save the updated CSV file
|
def main() -> None:
|
||||||
df.to_csv(file_path, sep=';', index=False)
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Seed or update databases.csv for backup configuration."
|
||||||
def main():
|
)
|
||||||
parser = argparse.ArgumentParser(description="Check and replace (or add) a database entry in a CSV file.")
|
parser.add_argument("file", help="Path to databases.csv")
|
||||||
parser.add_argument("file_path", help="Path to the CSV file")
|
parser.add_argument("instance", help="Instance name (e.g. bigbluebutton)")
|
||||||
parser.add_argument("instance", help="Database instance")
|
parser.add_argument(
|
||||||
parser.add_argument("database", help="Database name")
|
"database",
|
||||||
parser.add_argument("username", help="Username")
|
help="Database name or '*' to dump all databases",
|
||||||
parser.add_argument("password", nargs='?', default="", help="Password (optional)")
|
)
|
||||||
|
parser.add_argument("username", help="Database username")
|
||||||
|
parser.add_argument("password", help="Database password")
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
check_and_add_entry(args.file_path, args.instance, args.database, args.username, args.password)
|
try:
|
||||||
|
check_and_add_entry(
|
||||||
|
file_path=args.file,
|
||||||
|
instance=args.instance,
|
||||||
|
database=args.database,
|
||||||
|
username=args.username,
|
||||||
|
password=args.password,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"ERROR: {exc}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
@@ -34,7 +34,9 @@ def run(
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
def sh(cmd: str, *, capture: bool = True, check: bool = True) -> subprocess.CompletedProcess:
|
def sh(
|
||||||
|
cmd: str, *, capture: bool = True, check: bool = True
|
||||||
|
) -> subprocess.CompletedProcess:
|
||||||
return run(["sh", "-lc", cmd], capture=capture, check=check)
|
return run(["sh", "-lc", cmd], capture=capture, check=check)
|
||||||
|
|
||||||
|
|
||||||
@@ -63,24 +65,37 @@ def wait_for_log(container: str, pattern: str, timeout_s: int = 60) -> None:
|
|||||||
raise TimeoutError(f"Timed out waiting for log pattern '{pattern}' in {container}")
|
raise TimeoutError(f"Timed out waiting for log pattern '{pattern}' in {container}")
|
||||||
|
|
||||||
|
|
||||||
def wait_for_postgres(container: str, *, user: str = "postgres", timeout_s: int = 90) -> None:
|
def wait_for_postgres(
|
||||||
|
container: str, *, user: str = "postgres", timeout_s: int = 90
|
||||||
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Docker-outside-of-Docker friendly readiness: check from inside the DB container.
|
Docker-outside-of-Docker friendly readiness: check from inside the DB container.
|
||||||
"""
|
"""
|
||||||
deadline = time.time() + timeout_s
|
deadline = time.time() + timeout_s
|
||||||
while time.time() < deadline:
|
while time.time() < deadline:
|
||||||
p = run(
|
p = run(
|
||||||
["docker", "exec", container, "sh", "-lc", f"pg_isready -U {user} -h localhost"],
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
f"pg_isready -U {user} -h localhost",
|
||||||
|
],
|
||||||
capture=True,
|
capture=True,
|
||||||
check=False,
|
check=False,
|
||||||
)
|
)
|
||||||
if p.returncode == 0:
|
if p.returncode == 0:
|
||||||
return
|
return
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
raise TimeoutError(f"Timed out waiting for Postgres readiness in container {container}")
|
raise TimeoutError(
|
||||||
|
f"Timed out waiting for Postgres readiness in container {container}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def wait_for_mariadb(container: str, *, root_password: str, timeout_s: int = 90) -> None:
|
def wait_for_mariadb(
|
||||||
|
container: str, *, root_password: str, timeout_s: int = 90
|
||||||
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Liveness probe for MariaDB.
|
Liveness probe for MariaDB.
|
||||||
|
|
||||||
@@ -92,17 +107,28 @@ def wait_for_mariadb(container: str, *, root_password: str, timeout_s: int = 90)
|
|||||||
deadline = time.time() + timeout_s
|
deadline = time.time() + timeout_s
|
||||||
while time.time() < deadline:
|
while time.time() < deadline:
|
||||||
p = run(
|
p = run(
|
||||||
["docker", "exec", container, "sh", "-lc", "mariadb -uroot --protocol=socket -e \"SELECT 1;\""],
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
'mariadb -uroot --protocol=socket -e "SELECT 1;"',
|
||||||
|
],
|
||||||
capture=True,
|
capture=True,
|
||||||
check=False,
|
check=False,
|
||||||
)
|
)
|
||||||
if p.returncode == 0:
|
if p.returncode == 0:
|
||||||
return
|
return
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
raise TimeoutError(f"Timed out waiting for MariaDB readiness in container {container}")
|
raise TimeoutError(
|
||||||
|
f"Timed out waiting for MariaDB readiness in container {container}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def wait_for_mariadb_sql(container: str, *, user: str, password: str, timeout_s: int = 90) -> None:
|
def wait_for_mariadb_sql(
|
||||||
|
container: str, *, user: str, password: str, timeout_s: int = 90
|
||||||
|
) -> None:
|
||||||
"""
|
"""
|
||||||
SQL login readiness for the *dedicated test user* over TCP.
|
SQL login readiness for the *dedicated test user* over TCP.
|
||||||
|
|
||||||
@@ -118,7 +144,7 @@ def wait_for_mariadb_sql(container: str, *, user: str, password: str, timeout_s:
|
|||||||
container,
|
container,
|
||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{user} -p{password} -e \"SELECT 1;\"",
|
f'mariadb -h 127.0.0.1 -u{user} -p{password} -e "SELECT 1;"',
|
||||||
],
|
],
|
||||||
capture=True,
|
capture=True,
|
||||||
check=False,
|
check=False,
|
||||||
@@ -126,7 +152,9 @@ def wait_for_mariadb_sql(container: str, *, user: str, password: str, timeout_s:
|
|||||||
if p.returncode == 0:
|
if p.returncode == 0:
|
||||||
return
|
return
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
raise TimeoutError(f"Timed out waiting for MariaDB SQL login readiness in container {container}")
|
raise TimeoutError(
|
||||||
|
f"Timed out waiting for MariaDB SQL login readiness in container {container}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def backup_run(
|
def backup_run(
|
||||||
@@ -138,22 +166,29 @@ def backup_run(
|
|||||||
database_containers: list[str],
|
database_containers: list[str],
|
||||||
images_no_stop_required: list[str],
|
images_no_stop_required: list[str],
|
||||||
images_no_backup_required: list[str] | None = None,
|
images_no_backup_required: list[str] | None = None,
|
||||||
dump_only: bool = False,
|
dump_only_sql: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
cmd = [
|
cmd = [
|
||||||
"baudolo",
|
"baudolo",
|
||||||
"--compose-dir", compose_dir,
|
"--compose-dir",
|
||||||
"--docker-compose-hard-restart-required", "mailu",
|
compose_dir,
|
||||||
"--repo-name", repo_name,
|
"--docker-compose-hard-restart-required",
|
||||||
"--databases-csv", databases_csv,
|
"mailu",
|
||||||
"--backups-dir", backups_dir,
|
"--repo-name",
|
||||||
"--database-containers", *database_containers,
|
repo_name,
|
||||||
"--images-no-stop-required", *images_no_stop_required,
|
"--databases-csv",
|
||||||
|
databases_csv,
|
||||||
|
"--backups-dir",
|
||||||
|
backups_dir,
|
||||||
|
"--database-containers",
|
||||||
|
*database_containers,
|
||||||
|
"--images-no-stop-required",
|
||||||
|
*images_no_stop_required,
|
||||||
]
|
]
|
||||||
if images_no_backup_required:
|
if images_no_backup_required:
|
||||||
cmd += ["--images-no-backup-required", *images_no_backup_required]
|
cmd += ["--images-no-backup-required", *images_no_backup_required]
|
||||||
if dump_only:
|
if dump_only_sql:
|
||||||
cmd += ["--dump-only"]
|
cmd += ["--dump-only-sql"]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
run(cmd, capture=True, check=True)
|
run(cmd, capture=True, check=True)
|
||||||
|
|||||||
29
tests/e2e/test_e2e_cli_contract_dump_only_sql.py
Normal file
29
tests/e2e/test_e2e_cli_contract_dump_only_sql.py
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import run
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2ECLIContractDumpOnlySql(unittest.TestCase):
|
||||||
|
def test_help_mentions_new_flag(self) -> None:
|
||||||
|
cp = run(["baudolo", "--help"], capture=True, check=True)
|
||||||
|
out = (cp.stdout or "") + "\n" + (cp.stderr or "")
|
||||||
|
self.assertIn(
|
||||||
|
"--dump-only-sql",
|
||||||
|
out,
|
||||||
|
f"Expected '--dump-only-sql' to appear in --help output. Output:\n{out}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_old_flag_is_rejected(self) -> None:
|
||||||
|
cp = run(["baudolo", "--dump-only"], capture=True, check=False)
|
||||||
|
self.assertEqual(
|
||||||
|
cp.returncode,
|
||||||
|
2,
|
||||||
|
f"Expected exitcode 2 for unknown args, got {cp.returncode}\n"
|
||||||
|
f"STDOUT={cp.stdout}\nSTDERR={cp.stderr}",
|
||||||
|
)
|
||||||
|
err = (cp.stderr or "") + "\n" + (cp.stdout or "")
|
||||||
|
# Argparse typically prints "unrecognized arguments"
|
||||||
|
self.assertTrue(
|
||||||
|
("unrecognized arguments" in err) or ("usage:" in err.lower()),
|
||||||
|
f"Expected argparse-style error output. Output:\n{err}",
|
||||||
|
)
|
||||||
181
tests/e2e/test_e2e_dump_only_fallback_to_files.py
Normal file
181
tests/e2e/test_e2e_dump_only_fallback_to_files.py
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
# tests/e2e/test_e2e_dump_only_fallback_to_files.py
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
backup_path,
|
||||||
|
cleanup_docker,
|
||||||
|
create_minimal_compose_dir,
|
||||||
|
ensure_empty_dir,
|
||||||
|
latest_version_dir,
|
||||||
|
require_docker,
|
||||||
|
run,
|
||||||
|
unique,
|
||||||
|
write_databases_csv,
|
||||||
|
wait_for_postgres,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2EDumpOnlyFallbackToFiles(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
cls.prefix = unique("baudolo-e2e-dump-only-sql-fallback")
|
||||||
|
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||||
|
ensure_empty_dir(cls.backups_dir)
|
||||||
|
|
||||||
|
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||||
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
|
cls.pg_container = f"{cls.prefix}-pg"
|
||||||
|
cls.pg_volume = f"{cls.prefix}-pg-vol"
|
||||||
|
cls.restore_volume = f"{cls.prefix}-restore-vol"
|
||||||
|
|
||||||
|
cls.containers = [cls.pg_container]
|
||||||
|
cls.volumes = [cls.pg_volume, cls.restore_volume]
|
||||||
|
|
||||||
|
run(["docker", "volume", "create", cls.pg_volume])
|
||||||
|
|
||||||
|
# Start Postgres (creates a real DB volume)
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--name",
|
||||||
|
cls.pg_container,
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_PASSWORD=pgpw",
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_DB=appdb",
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_USER=postgres",
|
||||||
|
"-v",
|
||||||
|
f"{cls.pg_volume}:/var/lib/postgresql/data",
|
||||||
|
"postgres:16",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||||
|
|
||||||
|
# Add a deterministic marker file into the volume
|
||||||
|
cls.marker = "dump-only-sql-fallback-marker"
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
f"echo '{cls.marker}' > /var/lib/postgresql/data/marker.txt",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
# databases.csv WITHOUT matching entry for this instance -> should skip dump
|
||||||
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
|
write_databases_csv(cls.databases_csv, []) # empty except header
|
||||||
|
|
||||||
|
# Run baudolo with --dump-only-sql and a DB container present:
|
||||||
|
# Expected: WARNING + FALLBACK to file backup (files/ must exist)
|
||||||
|
cmd = [
|
||||||
|
"baudolo",
|
||||||
|
"--compose-dir",
|
||||||
|
cls.compose_dir,
|
||||||
|
"--docker-compose-hard-restart-required",
|
||||||
|
"mailu",
|
||||||
|
"--repo-name",
|
||||||
|
cls.repo_name,
|
||||||
|
"--databases-csv",
|
||||||
|
cls.databases_csv,
|
||||||
|
"--backups-dir",
|
||||||
|
cls.backups_dir,
|
||||||
|
"--database-containers",
|
||||||
|
cls.pg_container,
|
||||||
|
"--images-no-stop-required",
|
||||||
|
"postgres",
|
||||||
|
"mariadb",
|
||||||
|
"mysql",
|
||||||
|
"alpine",
|
||||||
|
"--dump-only-sql",
|
||||||
|
]
|
||||||
|
cp = run(cmd, capture=True, check=True)
|
||||||
|
|
||||||
|
cls.stdout = cp.stdout or ""
|
||||||
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
|
# Restore files into a fresh volume to prove file backup happened
|
||||||
|
run(["docker", "volume", "create", cls.restore_volume])
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"baudolo-restore",
|
||||||
|
"files",
|
||||||
|
cls.restore_volume,
|
||||||
|
cls.hash,
|
||||||
|
cls.version,
|
||||||
|
"--backups-dir",
|
||||||
|
cls.backups_dir,
|
||||||
|
"--repo-name",
|
||||||
|
cls.repo_name,
|
||||||
|
"--source-volume",
|
||||||
|
cls.pg_volume,
|
||||||
|
"--rsync-image",
|
||||||
|
"ghcr.io/kevinveenbirkenbach/alpine-rsync",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
|
def test_warns_about_missing_dump_in_dump_only_mode(self) -> None:
|
||||||
|
self.assertIn(
|
||||||
|
"WARNING: dump-only-sql requested but no DB dump was produced",
|
||||||
|
self.stdout,
|
||||||
|
f"Expected warning in baudolo output. STDOUT:\n{self.stdout}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_files_backup_exists_due_to_fallback(self) -> None:
|
||||||
|
p = (
|
||||||
|
backup_path(
|
||||||
|
self.backups_dir,
|
||||||
|
self.repo_name,
|
||||||
|
self.version,
|
||||||
|
self.pg_volume,
|
||||||
|
)
|
||||||
|
/ "files"
|
||||||
|
)
|
||||||
|
self.assertTrue(p.is_dir(), f"Expected files backup dir at: {p}")
|
||||||
|
|
||||||
|
def test_sql_dump_not_present(self) -> None:
|
||||||
|
# There should be no sql dumps because databases.csv had no matching entry.
|
||||||
|
sql_dir = (
|
||||||
|
backup_path(
|
||||||
|
self.backups_dir,
|
||||||
|
self.repo_name,
|
||||||
|
self.version,
|
||||||
|
self.pg_volume,
|
||||||
|
)
|
||||||
|
/ "sql"
|
||||||
|
)
|
||||||
|
# Could exist (dir created) in some edge cases, but should contain no *.sql dumps.
|
||||||
|
if sql_dir.exists():
|
||||||
|
dumps = list(sql_dir.glob("*.sql"))
|
||||||
|
self.assertEqual(
|
||||||
|
len(dumps),
|
||||||
|
0,
|
||||||
|
f"Did not expect SQL dump files, found: {dumps}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_restored_files_contain_marker(self) -> None:
|
||||||
|
p = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"-v",
|
||||||
|
f"{self.restore_volume}:/data",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"cat /data/marker.txt",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
self.assertEqual((p.stdout or "").strip(), self.marker)
|
||||||
184
tests/e2e/test_e2e_dump_only_sql_mixed_run.py
Normal file
184
tests/e2e/test_e2e_dump_only_sql_mixed_run.py
Normal file
@@ -0,0 +1,184 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
backup_path,
|
||||||
|
cleanup_docker,
|
||||||
|
create_minimal_compose_dir,
|
||||||
|
ensure_empty_dir,
|
||||||
|
latest_version_dir,
|
||||||
|
require_docker,
|
||||||
|
run,
|
||||||
|
unique,
|
||||||
|
wait_for_postgres,
|
||||||
|
write_databases_csv,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2EDumpOnlySqlMixedRun(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
cls.prefix = unique("baudolo-e2e-dump-only-sql-mixed-run")
|
||||||
|
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||||
|
ensure_empty_dir(cls.backups_dir)
|
||||||
|
|
||||||
|
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||||
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
|
# --- Volumes ---
|
||||||
|
cls.db_volume = f"{cls.prefix}-vol-db"
|
||||||
|
cls.files_volume = f"{cls.prefix}-vol-files"
|
||||||
|
|
||||||
|
# Track for cleanup
|
||||||
|
cls.containers: list[str] = []
|
||||||
|
cls.volumes = [cls.db_volume, cls.files_volume]
|
||||||
|
|
||||||
|
# Create volumes
|
||||||
|
run(["docker", "volume", "create", cls.db_volume])
|
||||||
|
run(["docker", "volume", "create", cls.files_volume])
|
||||||
|
|
||||||
|
# Put a marker into the non-db volume
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"-v",
|
||||||
|
f"{cls.files_volume}:/data",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"echo 'hello-non-db' > /data/hello.txt",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Start Postgres container using the DB volume ---
|
||||||
|
cls.pg_container = f"{cls.prefix}-pg"
|
||||||
|
cls.containers.append(cls.pg_container)
|
||||||
|
|
||||||
|
cls.pg_password = "postgres"
|
||||||
|
cls.pg_db = "testdb"
|
||||||
|
cls.pg_user = "postgres"
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--name",
|
||||||
|
cls.pg_container,
|
||||||
|
"-e",
|
||||||
|
f"POSTGRES_PASSWORD={cls.pg_password}",
|
||||||
|
"-v",
|
||||||
|
f"{cls.db_volume}:/var/lib/postgresql/data",
|
||||||
|
"postgres:16-alpine",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||||
|
|
||||||
|
# Create deterministic content in DB so dump is non-empty
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
f'psql -U postgres -c "CREATE DATABASE {cls.pg_db};" || true',
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
(
|
||||||
|
f"psql -U postgres -d {cls.pg_db} -c "
|
||||||
|
'"CREATE TABLE IF NOT EXISTS t (id INT PRIMARY KEY, v TEXT);'
|
||||||
|
"INSERT INTO t(id,v) VALUES (1,'hello-db') "
|
||||||
|
'ON CONFLICT (id) DO UPDATE SET v=EXCLUDED.v;"'
|
||||||
|
),
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# databases.csv with an entry => dump should succeed
|
||||||
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
|
write_databases_csv(
|
||||||
|
cls.databases_csv,
|
||||||
|
[(cls.pg_container, cls.pg_db, cls.pg_user, cls.pg_password)],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Run baudolo with dump-only-sql
|
||||||
|
cmd = [
|
||||||
|
"baudolo",
|
||||||
|
"--compose-dir",
|
||||||
|
cls.compose_dir,
|
||||||
|
"--databases-csv",
|
||||||
|
cls.databases_csv,
|
||||||
|
"--database-containers",
|
||||||
|
cls.pg_container,
|
||||||
|
"--images-no-stop-required",
|
||||||
|
"alpine",
|
||||||
|
"postgres",
|
||||||
|
"mariadb",
|
||||||
|
"mysql",
|
||||||
|
"--dump-only-sql",
|
||||||
|
"--backups-dir",
|
||||||
|
cls.backups_dir,
|
||||||
|
"--repo-name",
|
||||||
|
cls.repo_name,
|
||||||
|
]
|
||||||
|
cp = run(cmd, capture=True, check=True)
|
||||||
|
cls.stdout = cp.stdout
|
||||||
|
cls.stderr = cp.stderr
|
||||||
|
|
||||||
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
|
def test_db_volume_has_dump_and_no_files_dir(self) -> None:
|
||||||
|
base = backup_path(
|
||||||
|
self.backups_dir, self.repo_name, self.version, self.db_volume
|
||||||
|
)
|
||||||
|
|
||||||
|
dumps = base / "sql"
|
||||||
|
files = base / "files"
|
||||||
|
|
||||||
|
self.assertTrue(dumps.exists(), f"Expected dumps dir for DB volume at: {dumps}")
|
||||||
|
self.assertFalse(
|
||||||
|
files.exists(),
|
||||||
|
f"Did not expect files dir for DB volume when dump succeeded at: {files}",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Optional: at least one dump file exists
|
||||||
|
dump_files = list(dumps.glob("*.sql")) + list(dumps.glob("*.sql.gz"))
|
||||||
|
self.assertTrue(
|
||||||
|
dump_files,
|
||||||
|
f"Expected at least one SQL dump file in {dumps}, found none.",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_non_db_volume_has_files_dir(self) -> None:
|
||||||
|
base = backup_path(
|
||||||
|
self.backups_dir, self.repo_name, self.version, self.files_volume
|
||||||
|
)
|
||||||
|
files = base / "files"
|
||||||
|
self.assertTrue(
|
||||||
|
files.exists(),
|
||||||
|
f"Expected files dir for non-DB volume at: {files}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_dump_only_sql_does_not_disable_non_db_files_backup(self) -> None:
|
||||||
|
# Regression guard: even with --dump-only-sql, non-DB volumes must still be backed up as files
|
||||||
|
base = backup_path(
|
||||||
|
self.backups_dir, self.repo_name, self.version, self.files_volume
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
(base / "files").exists(),
|
||||||
|
f"Expected non-DB volume files backup to exist at: {base / 'files'}",
|
||||||
|
)
|
||||||
@@ -1,5 +1,4 @@
|
|||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from .helpers import (
|
from .helpers import (
|
||||||
backup_run,
|
backup_run,
|
||||||
@@ -33,12 +32,19 @@ class TestE2EFilesFull(unittest.TestCase):
|
|||||||
|
|
||||||
# create source volume with a file
|
# create source volume with a file
|
||||||
run(["docker", "volume", "create", cls.volume_src])
|
run(["docker", "volume", "create", cls.volume_src])
|
||||||
run([
|
run(
|
||||||
"docker", "run", "--rm",
|
[
|
||||||
"-v", f"{cls.volume_src}:/data",
|
"docker",
|
||||||
"alpine:3.20",
|
"run",
|
||||||
"sh", "-lc", "mkdir -p /data && echo 'hello' > /data/hello.txt",
|
"--rm",
|
||||||
])
|
"-v",
|
||||||
|
f"{cls.volume_src}:/data",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"mkdir -p /data && echo 'hello' > /data/hello.txt",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
# databases.csv (unused, but required by CLI)
|
# databases.csv (unused, but required by CLI)
|
||||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
@@ -75,20 +81,36 @@ class TestE2EFilesFull(unittest.TestCase):
|
|||||||
|
|
||||||
def test_restore_files_into_new_volume(self) -> None:
|
def test_restore_files_into_new_volume(self) -> None:
|
||||||
# restore files from volume_src backup into volume_dst
|
# restore files from volume_src backup into volume_dst
|
||||||
run([
|
run(
|
||||||
"baudolo-restore", "files",
|
[
|
||||||
self.volume_dst, self.hash, self.version,
|
"baudolo-restore",
|
||||||
"--backups-dir", self.backups_dir,
|
"files",
|
||||||
"--repo-name", self.repo_name,
|
self.volume_dst,
|
||||||
"--source-volume", self.volume_src,
|
self.hash,
|
||||||
"--rsync-image", "ghcr.io/kevinveenbirkenbach/alpine-rsync",
|
self.version,
|
||||||
])
|
"--backups-dir",
|
||||||
|
self.backups_dir,
|
||||||
|
"--repo-name",
|
||||||
|
self.repo_name,
|
||||||
|
"--source-volume",
|
||||||
|
self.volume_src,
|
||||||
|
"--rsync-image",
|
||||||
|
"ghcr.io/kevinveenbirkenbach/alpine-rsync",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
# verify restored file exists in dst volume
|
# verify restored file exists in dst volume
|
||||||
p = run([
|
p = run(
|
||||||
"docker", "run", "--rm",
|
[
|
||||||
"-v", f"{self.volume_dst}:/data",
|
"docker",
|
||||||
"alpine:3.20",
|
"run",
|
||||||
"sh", "-lc", "cat /data/hello.txt",
|
"--rm",
|
||||||
])
|
"-v",
|
||||||
|
f"{self.volume_dst}:/data",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"cat /data/hello.txt",
|
||||||
|
]
|
||||||
|
)
|
||||||
self.assertEqual((p.stdout or "").strip(), "hello")
|
self.assertEqual((p.stdout or "").strip(), "hello")
|
||||||
|
|||||||
@@ -26,22 +26,29 @@ class TestE2EFilesNoCopy(unittest.TestCase):
|
|||||||
cls.repo_name = cls.prefix
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
cls.volume_src = f"{cls.prefix}-vol-src"
|
cls.volume_src = f"{cls.prefix}-vol-src"
|
||||||
cls.volume_dst = f"{cls.prefix}-vol-dst"
|
cls.containers: list[str] = []
|
||||||
cls.containers = []
|
cls.volumes = [cls.volume_src]
|
||||||
cls.volumes = [cls.volume_src, cls.volume_dst]
|
|
||||||
|
|
||||||
|
# Create source volume and write a marker file
|
||||||
run(["docker", "volume", "create", cls.volume_src])
|
run(["docker", "volume", "create", cls.volume_src])
|
||||||
run([
|
run(
|
||||||
"docker", "run", "--rm",
|
[
|
||||||
"-v", f"{cls.volume_src}:/data",
|
"docker",
|
||||||
"alpine:3.20",
|
"run",
|
||||||
"sh", "-lc", "echo 'hello' > /data/hello.txt",
|
"--rm",
|
||||||
])
|
"-v",
|
||||||
|
f"{cls.volume_src}:/data",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"echo 'hello' > /data/hello.txt",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
write_databases_csv(cls.databases_csv, [])
|
write_databases_csv(cls.databases_csv, [])
|
||||||
|
|
||||||
# dump-only => NO file rsync backups
|
# dump-only-sql => non-DB volumes are STILL backed up as files
|
||||||
backup_run(
|
backup_run(
|
||||||
backups_dir=cls.backups_dir,
|
backups_dir=cls.backups_dir,
|
||||||
repo_name=cls.repo_name,
|
repo_name=cls.repo_name,
|
||||||
@@ -49,24 +56,64 @@ class TestE2EFilesNoCopy(unittest.TestCase):
|
|||||||
databases_csv=cls.databases_csv,
|
databases_csv=cls.databases_csv,
|
||||||
database_containers=["dummy-db"],
|
database_containers=["dummy-db"],
|
||||||
images_no_stop_required=["alpine", "postgres", "mariadb", "mysql"],
|
images_no_stop_required=["alpine", "postgres", "mariadb", "mysql"],
|
||||||
dump_only=True,
|
dump_only_sql=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
|
# Wipe the volume to ensure restore actually restores something
|
||||||
|
run(["docker", "volume", "rm", "-f", cls.volume_src])
|
||||||
|
run(["docker", "volume", "create", cls.volume_src])
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def tearDownClass(cls) -> None:
|
def tearDownClass(cls) -> None:
|
||||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
def test_files_backup_not_present(self) -> None:
|
def test_files_backup_present_for_non_db_volume(self) -> None:
|
||||||
p = backup_path(self.backups_dir, self.repo_name, self.version, self.volume_src) / "files"
|
p = (
|
||||||
self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
|
backup_path(self.backups_dir, self.repo_name, self.version, self.volume_src)
|
||||||
|
/ "files"
|
||||||
|
)
|
||||||
|
self.assertTrue(p.exists(), f"Expected files backup dir at: {p}")
|
||||||
|
|
||||||
def test_restore_files_fails_expected(self) -> None:
|
def test_restore_files_succeeds_and_restores_content(self) -> None:
|
||||||
p = run([
|
p = run(
|
||||||
"baudolo-restore", "files",
|
[
|
||||||
self.volume_dst, self.hash, self.version,
|
"baudolo-restore",
|
||||||
"--backups-dir", self.backups_dir,
|
"files",
|
||||||
"--repo-name", self.repo_name,
|
self.volume_src,
|
||||||
], check=False)
|
self.hash,
|
||||||
self.assertEqual(p.returncode, 2, f"Expected exitcode 2, got {p.returncode}\nSTDOUT={p.stdout}\nSTDERR={p.stderr}")
|
self.version,
|
||||||
|
"--backups-dir",
|
||||||
|
self.backups_dir,
|
||||||
|
"--repo-name",
|
||||||
|
self.repo_name,
|
||||||
|
],
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
p.returncode,
|
||||||
|
0,
|
||||||
|
f"Expected exitcode 0, got {p.returncode}\nSTDOUT={p.stdout}\nSTDERR={p.stderr}",
|
||||||
|
)
|
||||||
|
|
||||||
|
cp = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"-v",
|
||||||
|
f"{self.volume_src}:/data",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"cat /data/hello.txt",
|
||||||
|
],
|
||||||
|
capture=True,
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
cp.stdout.strip(),
|
||||||
|
"hello",
|
||||||
|
f"Unexpected restored content. STDOUT={cp.stdout}\nSTDERR={cp.stderr}",
|
||||||
|
)
|
||||||
|
|||||||
131
tests/e2e/test_e2e_images_no_backup_required_early_skip.py
Normal file
131
tests/e2e/test_e2e_images_no_backup_required_early_skip.py
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
# tests/e2e/test_e2e_images_no_backup_required_early_skip.py
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
backup_path,
|
||||||
|
cleanup_docker,
|
||||||
|
create_minimal_compose_dir,
|
||||||
|
ensure_empty_dir,
|
||||||
|
latest_version_dir,
|
||||||
|
require_docker,
|
||||||
|
run,
|
||||||
|
unique,
|
||||||
|
write_databases_csv,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2EImagesNoBackupRequiredEarlySkip(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
|
||||||
|
cls.prefix = unique("baudolo-e2e-early-skip-no-backup-required")
|
||||||
|
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||||
|
ensure_empty_dir(cls.backups_dir)
|
||||||
|
|
||||||
|
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||||
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
|
# --- Docker resources ---
|
||||||
|
cls.redis_container = f"{cls.prefix}-redis"
|
||||||
|
cls.ignored_volume = f"{cls.prefix}-redis-vol"
|
||||||
|
cls.normal_volume = f"{cls.prefix}-files-vol"
|
||||||
|
|
||||||
|
cls.containers = [cls.redis_container]
|
||||||
|
cls.volumes = [cls.ignored_volume, cls.normal_volume]
|
||||||
|
|
||||||
|
# Create volumes
|
||||||
|
run(["docker", "volume", "create", cls.ignored_volume])
|
||||||
|
run(["docker", "volume", "create", cls.normal_volume])
|
||||||
|
|
||||||
|
# Start redis container using the ignored volume
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--name",
|
||||||
|
cls.redis_container,
|
||||||
|
"-v",
|
||||||
|
f"{cls.ignored_volume}:/data",
|
||||||
|
"redis:alpine",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
# Put deterministic content into the normal volume
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"-v",
|
||||||
|
f"{cls.normal_volume}:/data",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"mkdir -p /data && echo 'hello' > /data/hello.txt",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
# databases.csv required by CLI (can be empty)
|
||||||
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
|
write_databases_csv(cls.databases_csv, [])
|
||||||
|
|
||||||
|
# Run baudolo with images-no-backup-required redis
|
||||||
|
cmd = [
|
||||||
|
"baudolo",
|
||||||
|
"--compose-dir",
|
||||||
|
cls.compose_dir,
|
||||||
|
"--docker-compose-hard-restart-required",
|
||||||
|
"mailu",
|
||||||
|
"--repo-name",
|
||||||
|
cls.repo_name,
|
||||||
|
"--databases-csv",
|
||||||
|
cls.databases_csv,
|
||||||
|
"--backups-dir",
|
||||||
|
cls.backups_dir,
|
||||||
|
"--database-containers",
|
||||||
|
"dummy-db",
|
||||||
|
"--images-no-stop-required",
|
||||||
|
"alpine",
|
||||||
|
"redis",
|
||||||
|
"postgres",
|
||||||
|
"mariadb",
|
||||||
|
"mysql",
|
||||||
|
"--images-no-backup-required",
|
||||||
|
"redis",
|
||||||
|
]
|
||||||
|
cp = run(cmd, capture=True, check=True)
|
||||||
|
cls.stdout = cp.stdout or ""
|
||||||
|
cls.stderr = cp.stderr or ""
|
||||||
|
|
||||||
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
|
def test_ignored_volume_has_no_backup_directory_at_all(self) -> None:
|
||||||
|
p = backup_path(
|
||||||
|
self.backups_dir,
|
||||||
|
self.repo_name,
|
||||||
|
self.version,
|
||||||
|
self.ignored_volume,
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
p.exists(),
|
||||||
|
f"Expected NO backup directory to be created for ignored volume, but found: {p}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_normal_volume_is_still_backed_up(self) -> None:
|
||||||
|
p = (
|
||||||
|
backup_path(
|
||||||
|
self.backups_dir,
|
||||||
|
self.repo_name,
|
||||||
|
self.version,
|
||||||
|
self.normal_volume,
|
||||||
|
)
|
||||||
|
/ "files"
|
||||||
|
/ "hello.txt"
|
||||||
|
)
|
||||||
|
self.assertTrue(p.is_file(), f"Expected backed up file at: {p}")
|
||||||
162
tests/e2e/test_e2e_mariadb_anonymous_preemption.py
Normal file
162
tests/e2e/test_e2e_mariadb_anonymous_preemption.py
Normal file
@@ -0,0 +1,162 @@
|
|||||||
|
"""
|
||||||
|
Bug-repro for: mariadb-dump fails with `ERROR 1045 Access denied for user
|
||||||
|
'<u>'@'localhost' (using password: YES)` when only '<u>'@'%' is granted and a
|
||||||
|
preempting ''@'localhost' user is present.
|
||||||
|
|
||||||
|
The fix forces TCP loopback in baudolo.backup.db so the dump matches the
|
||||||
|
'<u>'@'%' grant instead of the socket->localhost auth row.
|
||||||
|
|
||||||
|
This file:
|
||||||
|
- builds the exact preconditions that triggered the production failure,
|
||||||
|
- as a NEGATIVE control, runs a socket-based mariadb-dump (== the old code path)
|
||||||
|
and asserts that it fails with the literal 1045 / @'localhost' error,
|
||||||
|
- as a POSITIVE proof, calls backup_database() (where the fix lives) against
|
||||||
|
the same DB container and asserts the dump file is produced and contains the
|
||||||
|
seed data.
|
||||||
|
|
||||||
|
Note: the volume-rsync stage of baudolo is intentionally NOT exercised here.
|
||||||
|
That stage needs root on /var/lib/docker/volumes, which is provided by the
|
||||||
|
DinD wrapper in `make test-e2e` but not by an on-host invocation. The bug we
|
||||||
|
are verifying is in the DB-dump stage, so testing backup_database() directly
|
||||||
|
keeps the assertion focused and the test runnable both on-host and in DinD.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
import pandas
|
||||||
|
|
||||||
|
from baudolo.backup import db as db_mod
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
cleanup_docker,
|
||||||
|
require_docker,
|
||||||
|
run,
|
||||||
|
unique,
|
||||||
|
wait_for_mariadb,
|
||||||
|
wait_for_mariadb_sql,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2EMariaDBAnonymousPreemption(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
cls.prefix = unique("baudolo-e2e-mariadb-anon")
|
||||||
|
cls.db_container = f"{cls.prefix}-mariadb"
|
||||||
|
cls.db_volume = f"{cls.prefix}-mariadb-vol"
|
||||||
|
cls.containers = [cls.db_container]
|
||||||
|
cls.volumes = [cls.db_volume]
|
||||||
|
|
||||||
|
cls.db_name = "appdb"
|
||||||
|
cls.db_user = "tcponly"
|
||||||
|
cls.db_password = "tcponlypw"
|
||||||
|
cls.root_password = "rootpw"
|
||||||
|
|
||||||
|
run(["docker", "volume", "create", cls.db_volume])
|
||||||
|
|
||||||
|
# Boot WITHOUT MARIADB_USER/MARIADB_PASSWORD/MARIADB_DATABASE so the
|
||||||
|
# entrypoint does not auto-create '<u>'@'%'. We provision the user
|
||||||
|
# explicitly below to mirror the SQL path used by svc-db-mariadb.
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--name",
|
||||||
|
cls.db_container,
|
||||||
|
"-e",
|
||||||
|
f"MARIADB_ROOT_PASSWORD={cls.root_password}",
|
||||||
|
"-v",
|
||||||
|
f"{cls.db_volume}:/var/lib/mysql",
|
||||||
|
"mariadb:12.2",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
wait_for_mariadb(
|
||||||
|
cls.db_container, root_password=cls.root_password, timeout_s=120
|
||||||
|
)
|
||||||
|
|
||||||
|
# Provision: '<u>'@'%' (the app/backup grant) + anonymous ''@'localhost'
|
||||||
|
# (the preemption trigger). Mirrors the production state that produced
|
||||||
|
# `ERROR 1045 ... '<u>'@'localhost' (using password: YES)`.
|
||||||
|
bootstrap_sql = (
|
||||||
|
f"CREATE DATABASE {cls.db_name};"
|
||||||
|
f"CREATE USER '{cls.db_user}'@'%' IDENTIFIED BY '{cls.db_password}';"
|
||||||
|
f"GRANT ALL PRIVILEGES ON {cls.db_name}.* TO '{cls.db_user}'@'%';"
|
||||||
|
f"CREATE USER ''@'localhost' IDENTIFIED BY 'anonpw-not-{cls.db_password}';"
|
||||||
|
"FLUSH PRIVILEGES;"
|
||||||
|
f"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50));"
|
||||||
|
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');"
|
||||||
|
)
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
cls.db_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
f'mariadb -uroot --protocol=socket -e "{bootstrap_sql}"',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
# Sanity: '<u>' can log in over TCP (matches '%'). If THIS fails,
|
||||||
|
# the precondition for the fix to even apply is broken.
|
||||||
|
wait_for_mariadb_sql(
|
||||||
|
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=60
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
|
def test_negative_control_socket_dump_fails_with_1045(self) -> None:
|
||||||
|
# Reproduces the OLD code path (no -h/--protocol). MUST fail with 1045
|
||||||
|
# under the configured preemption. If this ever starts passing, either
|
||||||
|
# the MariaDB auth semantics changed or the anonymous-user setup did
|
||||||
|
# not take effect — in both cases the positive test below loses its
|
||||||
|
# ability to discriminate "fix works" vs "bug never reproduced".
|
||||||
|
p = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
self.db_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
f"mariadb-dump -u{self.db_user} -p{self.db_password} {self.db_name}",
|
||||||
|
],
|
||||||
|
capture=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
self.assertNotEqual(p.returncode, 0, "socket-based dump unexpectedly succeeded")
|
||||||
|
self.assertIn("1045", (p.stderr or "") + (p.stdout or ""))
|
||||||
|
self.assertIn("@'localhost'", (p.stderr or "") + (p.stdout or ""))
|
||||||
|
|
||||||
|
def test_backup_database_succeeds_with_tcp_fix(self) -> None:
|
||||||
|
# Drives the function where the fix lives. No rsync, no privileged
|
||||||
|
# paths — just the dump that the negative-control proved is failing
|
||||||
|
# under the same preemption setup.
|
||||||
|
with tempfile.TemporaryDirectory() as volume_dir:
|
||||||
|
df = pandas.DataFrame(
|
||||||
|
[(self.db_container, self.db_name, self.db_user, self.db_password)],
|
||||||
|
columns=["instance", "database", "username", "password"],
|
||||||
|
)
|
||||||
|
produced = db_mod.backup_database(
|
||||||
|
container=self.db_container,
|
||||||
|
volume_dir=volume_dir,
|
||||||
|
db_type="mariadb",
|
||||||
|
databases_df=df,
|
||||||
|
database_containers=[self.db_container],
|
||||||
|
)
|
||||||
|
self.assertTrue(produced, "backup_database did not produce a dump")
|
||||||
|
dump_path = os.path.join(volume_dir, "sql", f"{self.db_name}.backup.sql")
|
||||||
|
self.assertTrue(os.path.isfile(dump_path), f"expected dump at {dump_path}")
|
||||||
|
with open(dump_path, "r", encoding="utf-8", errors="replace") as f:
|
||||||
|
content = f.read()
|
||||||
|
self.assertIn("INSERT INTO", content)
|
||||||
|
self.assertIn("'ok'", content)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
@@ -62,8 +62,12 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Liveness + actual SQL login readiness (TCP)
|
# Liveness + actual SQL login readiness (TCP)
|
||||||
wait_for_mariadb(cls.db_container, root_password=cls.root_password, timeout_s=90)
|
wait_for_mariadb(
|
||||||
wait_for_mariadb_sql(cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90)
|
cls.db_container, root_password=cls.root_password, timeout_s=90
|
||||||
|
)
|
||||||
|
wait_for_mariadb_sql(
|
||||||
|
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||||
|
)
|
||||||
|
|
||||||
# Create table + data via the dedicated user (TCP)
|
# Create table + data via the dedicated user (TCP)
|
||||||
run(
|
run(
|
||||||
@@ -74,14 +78,17 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
f"-e \"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); "
|
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
# IMPORTANT: baudolo backup expects credentials for the DB dump.
|
# IMPORTANT: baudolo backup expects credentials for the DB dump.
|
||||||
write_databases_csv(cls.databases_csv, [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)])
|
write_databases_csv(
|
||||||
|
cls.databases_csv,
|
||||||
|
[(cls.db_container, cls.db_name, cls.db_user, cls.db_password)],
|
||||||
|
)
|
||||||
|
|
||||||
# Backup with file+dump
|
# Backup with file+dump
|
||||||
backup_run(
|
backup_run(
|
||||||
@@ -104,7 +111,7 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
f"-e \"DROP TABLE {cls.db_name}.t;\"",
|
f'-e "DROP TABLE {cls.db_name}.t;"',
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -137,7 +144,11 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
def test_dump_file_exists(self) -> None:
|
def test_dump_file_exists(self) -> None:
|
||||||
p = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume) / "sql" / f"{self.db_name}.backup.sql"
|
p = (
|
||||||
|
backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume)
|
||||||
|
/ "sql"
|
||||||
|
/ f"{self.db_name}.backup.sql"
|
||||||
|
)
|
||||||
self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
|
self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
|
||||||
|
|
||||||
def test_data_restored(self) -> None:
|
def test_data_restored(self) -> None:
|
||||||
@@ -149,7 +160,7 @@ class TestE2EMariaDBFull(unittest.TestCase):
|
|||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||||
f"-N -e \"SELECT v FROM {self.db_name}.t WHERE id=1;\"",
|
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||||
|
|||||||
@@ -60,8 +60,12 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
wait_for_mariadb(cls.db_container, root_password=cls.root_password, timeout_s=90)
|
wait_for_mariadb(
|
||||||
wait_for_mariadb_sql(cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90)
|
cls.db_container, root_password=cls.root_password, timeout_s=90
|
||||||
|
)
|
||||||
|
wait_for_mariadb_sql(
|
||||||
|
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90
|
||||||
|
)
|
||||||
|
|
||||||
# Create table + data (TCP)
|
# Create table + data (TCP)
|
||||||
run(
|
run(
|
||||||
@@ -72,15 +76,18 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
f"-e \"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); "
|
f'-e "CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); '
|
||||||
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
write_databases_csv(cls.databases_csv, [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)])
|
write_databases_csv(
|
||||||
|
cls.databases_csv,
|
||||||
|
[(cls.db_container, cls.db_name, cls.db_user, cls.db_password)],
|
||||||
|
)
|
||||||
|
|
||||||
# dump-only => no files
|
# dump-only-sql => no files
|
||||||
backup_run(
|
backup_run(
|
||||||
backups_dir=cls.backups_dir,
|
backups_dir=cls.backups_dir,
|
||||||
repo_name=cls.repo_name,
|
repo_name=cls.repo_name,
|
||||||
@@ -88,7 +95,7 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
databases_csv=cls.databases_csv,
|
databases_csv=cls.databases_csv,
|
||||||
database_containers=[cls.db_container],
|
database_containers=[cls.db_container],
|
||||||
images_no_stop_required=["mariadb", "mysql", "alpine", "postgres"],
|
images_no_stop_required=["mariadb", "mysql", "alpine", "postgres"],
|
||||||
dump_only=True,
|
dump_only_sql=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
@@ -102,7 +109,7 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
|
||||||
f"-e \"DROP TABLE {cls.db_name}.t;\"",
|
f'-e "DROP TABLE {cls.db_name}.t;"',
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -135,7 +142,10 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
def test_files_backup_not_present(self) -> None:
|
def test_files_backup_not_present(self) -> None:
|
||||||
p = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume) / "files"
|
p = (
|
||||||
|
backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume)
|
||||||
|
/ "files"
|
||||||
|
)
|
||||||
self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
|
self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
|
||||||
|
|
||||||
def test_data_restored(self) -> None:
|
def test_data_restored(self) -> None:
|
||||||
@@ -147,7 +157,7 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
|
|||||||
"sh",
|
"sh",
|
||||||
"-lc",
|
"-lc",
|
||||||
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
|
||||||
f"-N -e \"SELECT v FROM {self.db_name}.t WHERE id=1;\"",
|
f'-N -e "SELECT v FROM {self.db_name}.t WHERE id=1;"',
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||||
|
|||||||
@@ -33,26 +33,42 @@ class TestE2EPostgresFull(unittest.TestCase):
|
|||||||
|
|
||||||
run(["docker", "volume", "create", cls.pg_volume])
|
run(["docker", "volume", "create", cls.pg_volume])
|
||||||
|
|
||||||
run([
|
run(
|
||||||
"docker", "run", "-d",
|
[
|
||||||
"--name", cls.pg_container,
|
"docker",
|
||||||
"-e", "POSTGRES_PASSWORD=pgpw",
|
"run",
|
||||||
"-e", "POSTGRES_DB=appdb",
|
"-d",
|
||||||
"-e", "POSTGRES_USER=postgres",
|
"--name",
|
||||||
"-v", f"{cls.pg_volume}:/var/lib/postgresql/data",
|
cls.pg_container,
|
||||||
"postgres:16",
|
"-e",
|
||||||
])
|
"POSTGRES_PASSWORD=pgpw",
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_DB=appdb",
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_USER=postgres",
|
||||||
|
"-v",
|
||||||
|
f"{cls.pg_volume}:/var/lib/postgresql/data",
|
||||||
|
"postgres:16",
|
||||||
|
]
|
||||||
|
)
|
||||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||||
|
|
||||||
# Create a table + data
|
# Create a table + data
|
||||||
run([
|
run(
|
||||||
"docker", "exec", cls.pg_container,
|
[
|
||||||
"sh", "-lc",
|
"docker",
|
||||||
"psql -U postgres -d appdb -c \"CREATE TABLE t (id int primary key, v text); INSERT INTO t VALUES (1,'ok');\"",
|
"exec",
|
||||||
])
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"psql -U postgres -d appdb -c \"CREATE TABLE t (id int primary key, v text); INSERT INTO t VALUES (1,'ok');\"",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
write_databases_csv(cls.databases_csv, [(cls.pg_container, "appdb", "postgres", "pgpw")])
|
write_databases_csv(
|
||||||
|
cls.databases_csv, [(cls.pg_container, "appdb", "postgres", "pgpw")]
|
||||||
|
)
|
||||||
|
|
||||||
backup_run(
|
backup_run(
|
||||||
backups_dir=cls.backups_dir,
|
backups_dir=cls.backups_dir,
|
||||||
@@ -66,37 +82,62 @@ class TestE2EPostgresFull(unittest.TestCase):
|
|||||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
# Wipe schema
|
# Wipe schema
|
||||||
run([
|
run(
|
||||||
"docker", "exec", cls.pg_container,
|
[
|
||||||
"sh", "-lc",
|
"docker",
|
||||||
"psql -U postgres -d appdb -c \"DROP TABLE t;\"",
|
"exec",
|
||||||
])
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
'psql -U postgres -d appdb -c "DROP TABLE t;"',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
# Restore
|
# Restore
|
||||||
run([
|
run(
|
||||||
"baudolo-restore", "postgres",
|
[
|
||||||
cls.pg_volume, cls.hash, cls.version,
|
"baudolo-restore",
|
||||||
"--backups-dir", cls.backups_dir,
|
"postgres",
|
||||||
"--repo-name", cls.repo_name,
|
cls.pg_volume,
|
||||||
"--container", cls.pg_container,
|
cls.hash,
|
||||||
"--db-name", "appdb",
|
cls.version,
|
||||||
"--db-user", "postgres",
|
"--backups-dir",
|
||||||
"--db-password", "pgpw",
|
cls.backups_dir,
|
||||||
"--empty",
|
"--repo-name",
|
||||||
])
|
cls.repo_name,
|
||||||
|
"--container",
|
||||||
|
cls.pg_container,
|
||||||
|
"--db-name",
|
||||||
|
"appdb",
|
||||||
|
"--db-user",
|
||||||
|
"postgres",
|
||||||
|
"--db-password",
|
||||||
|
"pgpw",
|
||||||
|
"--empty",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def tearDownClass(cls) -> None:
|
def tearDownClass(cls) -> None:
|
||||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
def test_dump_file_exists(self) -> None:
|
def test_dump_file_exists(self) -> None:
|
||||||
p = backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume) / "sql" / "appdb.backup.sql"
|
p = (
|
||||||
|
backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume)
|
||||||
|
/ "sql"
|
||||||
|
/ "appdb.backup.sql"
|
||||||
|
)
|
||||||
self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
|
self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
|
||||||
|
|
||||||
def test_data_restored(self) -> None:
|
def test_data_restored(self) -> None:
|
||||||
p = run([
|
p = run(
|
||||||
"docker", "exec", self.pg_container,
|
[
|
||||||
"sh", "-lc",
|
"docker",
|
||||||
"psql -U postgres -d appdb -t -c \"SELECT v FROM t WHERE id=1;\"",
|
"exec",
|
||||||
])
|
self.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
'psql -U postgres -d appdb -t -c "SELECT v FROM t WHERE id=1;"',
|
||||||
|
]
|
||||||
|
)
|
||||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||||
|
|||||||
@@ -32,25 +32,41 @@ class TestE2EPostgresNoCopy(unittest.TestCase):
|
|||||||
cls.volumes = [cls.pg_volume]
|
cls.volumes = [cls.pg_volume]
|
||||||
|
|
||||||
run(["docker", "volume", "create", cls.pg_volume])
|
run(["docker", "volume", "create", cls.pg_volume])
|
||||||
run([
|
run(
|
||||||
"docker", "run", "-d",
|
[
|
||||||
"--name", cls.pg_container,
|
"docker",
|
||||||
"-e", "POSTGRES_PASSWORD=pgpw",
|
"run",
|
||||||
"-e", "POSTGRES_DB=appdb",
|
"-d",
|
||||||
"-e", "POSTGRES_USER=postgres",
|
"--name",
|
||||||
"-v", f"{cls.pg_volume}:/var/lib/postgresql/data",
|
cls.pg_container,
|
||||||
"postgres:16",
|
"-e",
|
||||||
])
|
"POSTGRES_PASSWORD=pgpw",
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_DB=appdb",
|
||||||
|
"-e",
|
||||||
|
"POSTGRES_USER=postgres",
|
||||||
|
"-v",
|
||||||
|
f"{cls.pg_volume}:/var/lib/postgresql/data",
|
||||||
|
"postgres:16",
|
||||||
|
]
|
||||||
|
)
|
||||||
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||||
|
|
||||||
run([
|
run(
|
||||||
"docker", "exec", cls.pg_container,
|
[
|
||||||
"sh", "-lc",
|
"docker",
|
||||||
"psql -U postgres -d appdb -c \"CREATE TABLE t (id int primary key, v text); INSERT INTO t VALUES (1,'ok');\"",
|
"exec",
|
||||||
])
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"psql -U postgres -d appdb -c \"CREATE TABLE t (id int primary key, v text); INSERT INTO t VALUES (1,'ok');\"",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
write_databases_csv(cls.databases_csv, [(cls.pg_container, "appdb", "postgres", "pgpw")])
|
write_databases_csv(
|
||||||
|
cls.databases_csv, [(cls.pg_container, "appdb", "postgres", "pgpw")]
|
||||||
|
)
|
||||||
|
|
||||||
backup_run(
|
backup_run(
|
||||||
backups_dir=cls.backups_dir,
|
backups_dir=cls.backups_dir,
|
||||||
@@ -59,41 +75,65 @@ class TestE2EPostgresNoCopy(unittest.TestCase):
|
|||||||
databases_csv=cls.databases_csv,
|
databases_csv=cls.databases_csv,
|
||||||
database_containers=[cls.pg_container],
|
database_containers=[cls.pg_container],
|
||||||
images_no_stop_required=["postgres", "mariadb", "mysql", "alpine"],
|
images_no_stop_required=["postgres", "mariadb", "mysql", "alpine"],
|
||||||
dump_only=True,
|
dump_only_sql=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
run([
|
run(
|
||||||
"docker", "exec", cls.pg_container,
|
[
|
||||||
"sh", "-lc",
|
"docker",
|
||||||
"psql -U postgres -d appdb -c \"DROP TABLE t;\"",
|
"exec",
|
||||||
])
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
'psql -U postgres -d appdb -c "DROP TABLE t;"',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
run([
|
run(
|
||||||
"baudolo-restore", "postgres",
|
[
|
||||||
cls.pg_volume, cls.hash, cls.version,
|
"baudolo-restore",
|
||||||
"--backups-dir", cls.backups_dir,
|
"postgres",
|
||||||
"--repo-name", cls.repo_name,
|
cls.pg_volume,
|
||||||
"--container", cls.pg_container,
|
cls.hash,
|
||||||
"--db-name", "appdb",
|
cls.version,
|
||||||
"--db-user", "postgres",
|
"--backups-dir",
|
||||||
"--db-password", "pgpw",
|
cls.backups_dir,
|
||||||
"--empty",
|
"--repo-name",
|
||||||
])
|
cls.repo_name,
|
||||||
|
"--container",
|
||||||
|
cls.pg_container,
|
||||||
|
"--db-name",
|
||||||
|
"appdb",
|
||||||
|
"--db-user",
|
||||||
|
"postgres",
|
||||||
|
"--db-password",
|
||||||
|
"pgpw",
|
||||||
|
"--empty",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def tearDownClass(cls) -> None:
|
def tearDownClass(cls) -> None:
|
||||||
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
def test_files_backup_not_present(self) -> None:
|
def test_files_backup_not_present(self) -> None:
|
||||||
p = backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume) / "files"
|
p = (
|
||||||
|
backup_path(self.backups_dir, self.repo_name, self.version, self.pg_volume)
|
||||||
|
/ "files"
|
||||||
|
)
|
||||||
self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
|
self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
|
||||||
|
|
||||||
def test_data_restored(self) -> None:
|
def test_data_restored(self) -> None:
|
||||||
p = run([
|
p = run(
|
||||||
"docker", "exec", self.pg_container,
|
[
|
||||||
"sh", "-lc",
|
"docker",
|
||||||
"psql -U postgres -d appdb -t -c \"SELECT v FROM t WHERE id=1;\"",
|
"exec",
|
||||||
])
|
self.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
'psql -U postgres -d appdb -t -c "SELECT v FROM t WHERE id=1;"',
|
||||||
|
]
|
||||||
|
)
|
||||||
self.assertEqual((p.stdout or "").strip(), "ok")
|
self.assertEqual((p.stdout or "").strip(), "ok")
|
||||||
|
|||||||
232
tests/e2e/test_e2e_seed_star_and_db_entries_backup_postgres.py
Normal file
232
tests/e2e/test_e2e_seed_star_and_db_entries_backup_postgres.py
Normal file
@@ -0,0 +1,232 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
backup_path,
|
||||||
|
cleanup_docker,
|
||||||
|
create_minimal_compose_dir,
|
||||||
|
ensure_empty_dir,
|
||||||
|
latest_version_dir,
|
||||||
|
require_docker,
|
||||||
|
run,
|
||||||
|
unique,
|
||||||
|
wait_for_postgres,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2ESeedStarAndDbEntriesBackupPostgres(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
|
||||||
|
cls.prefix = unique("baudolo-e2e-seed-star-and-db")
|
||||||
|
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||||
|
ensure_empty_dir(cls.backups_dir)
|
||||||
|
|
||||||
|
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||||
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
|
# --- Volumes ---
|
||||||
|
cls.db_volume = f"{cls.prefix}-vol-db"
|
||||||
|
cls.files_volume = f"{cls.prefix}-vol-files"
|
||||||
|
cls.volumes = [cls.db_volume, cls.files_volume]
|
||||||
|
|
||||||
|
run(["docker", "volume", "create", cls.db_volume])
|
||||||
|
run(["docker", "volume", "create", cls.files_volume])
|
||||||
|
|
||||||
|
# Put a marker into the non-db volume
|
||||||
|
cls.marker = "hello-non-db-seed-star"
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"-v",
|
||||||
|
f"{cls.files_volume}:/data",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
f"echo '{cls.marker}' > /data/hello.txt",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Start Postgres container using the DB volume ---
|
||||||
|
cls.pg_container = f"{cls.prefix}-pg"
|
||||||
|
cls.containers = [cls.pg_container]
|
||||||
|
|
||||||
|
cls.pg_password = "postgres"
|
||||||
|
cls.pg_user = "postgres"
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"-d",
|
||||||
|
"--name",
|
||||||
|
cls.pg_container,
|
||||||
|
"-e",
|
||||||
|
f"POSTGRES_PASSWORD={cls.pg_password}",
|
||||||
|
"-v",
|
||||||
|
f"{cls.db_volume}:/var/lib/postgresql/data",
|
||||||
|
"postgres:16-alpine",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
wait_for_postgres(cls.pg_container, user="postgres", timeout_s=90)
|
||||||
|
|
||||||
|
# Create two DBs and deterministic content, so pg_dumpall is meaningful
|
||||||
|
cls.pg_db1 = "testdb1"
|
||||||
|
cls.pg_db2 = "testdb2"
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
(
|
||||||
|
f'psql -U {cls.pg_user} -c "CREATE DATABASE {cls.pg_db1};" || true; '
|
||||||
|
f'psql -U {cls.pg_user} -c "CREATE DATABASE {cls.pg_db2};" || true; '
|
||||||
|
),
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
(
|
||||||
|
f"psql -U {cls.pg_user} -d {cls.pg_db1} -c "
|
||||||
|
'"CREATE TABLE IF NOT EXISTS t (id INT PRIMARY KEY, v TEXT);'
|
||||||
|
"INSERT INTO t(id,v) VALUES (1,'hello-db1') "
|
||||||
|
'ON CONFLICT (id) DO UPDATE SET v=EXCLUDED.v;"'
|
||||||
|
),
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"exec",
|
||||||
|
cls.pg_container,
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
(
|
||||||
|
f"psql -U {cls.pg_user} -d {cls.pg_db2} -c "
|
||||||
|
'"CREATE TABLE IF NOT EXISTS t (id INT PRIMARY KEY, v TEXT);'
|
||||||
|
"INSERT INTO t(id,v) VALUES (1,'hello-db2') "
|
||||||
|
'ON CONFLICT (id) DO UPDATE SET v=EXCLUDED.v;"'
|
||||||
|
),
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Seed databases.csv using CLI (star + concrete db) ---
|
||||||
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
|
|
||||||
|
# IMPORTANT: because we pass --database-containers <container>,
|
||||||
|
# get_instance() will use the container name as instance key.
|
||||||
|
instance = cls.pg_container
|
||||||
|
|
||||||
|
# Seed star entry (pg_dumpall)
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"baudolo-seed",
|
||||||
|
cls.databases_csv,
|
||||||
|
instance,
|
||||||
|
"*",
|
||||||
|
cls.pg_user,
|
||||||
|
cls.pg_password,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
# Seed concrete DB entry (pg_dump)
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"baudolo-seed",
|
||||||
|
cls.databases_csv,
|
||||||
|
instance,
|
||||||
|
cls.pg_db1,
|
||||||
|
cls.pg_user,
|
||||||
|
cls.pg_password,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Run baudolo with dump-only-sql ---
|
||||||
|
cmd = [
|
||||||
|
"baudolo",
|
||||||
|
"--compose-dir",
|
||||||
|
cls.compose_dir,
|
||||||
|
"--databases-csv",
|
||||||
|
cls.databases_csv,
|
||||||
|
"--database-containers",
|
||||||
|
cls.pg_container,
|
||||||
|
"--images-no-stop-required",
|
||||||
|
"alpine",
|
||||||
|
"postgres",
|
||||||
|
"mariadb",
|
||||||
|
"mysql",
|
||||||
|
"--dump-only-sql",
|
||||||
|
"--backups-dir",
|
||||||
|
cls.backups_dir,
|
||||||
|
"--repo-name",
|
||||||
|
cls.repo_name,
|
||||||
|
]
|
||||||
|
cp = run(cmd, capture=True, check=True)
|
||||||
|
cls.stdout = cp.stdout or ""
|
||||||
|
cls.stderr = cp.stderr or ""
|
||||||
|
|
||||||
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
|
||||||
|
|
||||||
|
def test_db_volume_has_cluster_dump_and_concrete_db_dump_and_no_files(self) -> None:
|
||||||
|
base = backup_path(
|
||||||
|
self.backups_dir, self.repo_name, self.version, self.db_volume
|
||||||
|
)
|
||||||
|
sql_dir = base / "sql"
|
||||||
|
files_dir = base / "files"
|
||||||
|
|
||||||
|
self.assertTrue(sql_dir.exists(), f"Expected sql dir at: {sql_dir}")
|
||||||
|
self.assertFalse(
|
||||||
|
files_dir.exists(),
|
||||||
|
f"Did not expect files dir for DB volume when dump-only-sql succeeded: {files_dir}",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Cluster dump file produced by '*' entry
|
||||||
|
cluster = sql_dir / f"{self.pg_container}.cluster.backup.sql"
|
||||||
|
self.assertTrue(cluster.is_file(), f"Expected cluster dump file at: {cluster}")
|
||||||
|
|
||||||
|
# Concrete DB dump produced by normal entry
|
||||||
|
db1 = sql_dir / f"{self.pg_db1}.backup.sql"
|
||||||
|
self.assertTrue(db1.is_file(), f"Expected db dump file at: {db1}")
|
||||||
|
|
||||||
|
# Basic sanity: cluster dump usually contains CREATE DATABASE statements
|
||||||
|
txt = cluster.read_text(encoding="utf-8", errors="ignore")
|
||||||
|
self.assertIn(
|
||||||
|
"CREATE DATABASE",
|
||||||
|
txt,
|
||||||
|
"Expected cluster dump to contain CREATE DATABASE statements",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_non_db_volume_still_has_files_backup(self) -> None:
|
||||||
|
base = backup_path(
|
||||||
|
self.backups_dir, self.repo_name, self.version, self.files_volume
|
||||||
|
)
|
||||||
|
files_dir = base / "files"
|
||||||
|
|
||||||
|
self.assertTrue(
|
||||||
|
files_dir.exists(), f"Expected files dir for non-DB volume at: {files_dir}"
|
||||||
|
)
|
||||||
|
|
||||||
|
marker = files_dir / "hello.txt"
|
||||||
|
self.assertTrue(marker.is_file(), f"Expected marker file at: {marker}")
|
||||||
|
self.assertEqual(
|
||||||
|
marker.read_text(encoding="utf-8").strip(),
|
||||||
|
self.marker,
|
||||||
|
)
|
||||||
196
tests/e2e/test_e2e_swarm_task_skip.py
Normal file
196
tests/e2e/test_e2e_swarm_task_skip.py
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
# tests/e2e/test_e2e_swarm_task_skip.py
|
||||||
|
#
|
||||||
|
# Reproduces the swarm flake fixed on this branch: baudolo used to stop a
|
||||||
|
# swarm task container around the volume file backup because its image was
|
||||||
|
# not whitelisted; the orchestrator immediately replaced the stopped task and
|
||||||
|
# the later `docker start` failed on the detached overlay network, killing
|
||||||
|
# the backup run. With the fix the task container is skipped (backed up hot):
|
||||||
|
# the backup succeeds, the very same container instance keeps running, and
|
||||||
|
# the service never has to replace a task.
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from .helpers import (
|
||||||
|
backup_path,
|
||||||
|
backup_run,
|
||||||
|
create_minimal_compose_dir,
|
||||||
|
ensure_empty_dir,
|
||||||
|
latest_version_dir,
|
||||||
|
require_docker,
|
||||||
|
run,
|
||||||
|
unique,
|
||||||
|
write_databases_csv,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _swarm_state() -> str:
|
||||||
|
return run(
|
||||||
|
["docker", "info", "--format", "{{.Swarm.LocalNodeState}}"]
|
||||||
|
).stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _task_container_id(service: str, timeout_s: int = 60) -> str:
|
||||||
|
deadline = time.time() + timeout_s
|
||||||
|
while time.time() < deadline:
|
||||||
|
out = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"ps",
|
||||||
|
"--filter",
|
||||||
|
f"label=com.docker.swarm.service.name={service}",
|
||||||
|
"--format",
|
||||||
|
"{{.ID}}",
|
||||||
|
]
|
||||||
|
).stdout.strip()
|
||||||
|
if out:
|
||||||
|
return out.splitlines()[0]
|
||||||
|
time.sleep(2)
|
||||||
|
raise RuntimeError(f"No running task container for service {service}")
|
||||||
|
|
||||||
|
|
||||||
|
def _started_at(container_id: str) -> str:
|
||||||
|
return run(
|
||||||
|
["docker", "inspect", "--format", "{{.State.StartedAt}}", container_id]
|
||||||
|
).stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
class TestE2ESwarmTaskSkip(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
require_docker()
|
||||||
|
cls.prefix = unique("baudolo-e2e-swarm-skip")
|
||||||
|
cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
|
||||||
|
ensure_empty_dir(cls.backups_dir)
|
||||||
|
cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
|
||||||
|
cls.repo_name = cls.prefix
|
||||||
|
|
||||||
|
cls.swarm_initted = False
|
||||||
|
if _swarm_state() != "active":
|
||||||
|
run(["docker", "swarm", "init", "--advertise-addr", "127.0.0.1"])
|
||||||
|
cls.swarm_initted = True
|
||||||
|
|
||||||
|
cls.volume = f"{cls.prefix}-vol"
|
||||||
|
cls.service = f"{cls.prefix}-svc"
|
||||||
|
cls.volumes = [cls.volume]
|
||||||
|
|
||||||
|
run(["docker", "volume", "create", cls.volume])
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"-v",
|
||||||
|
f"{cls.volume}:/data",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sh",
|
||||||
|
"-lc",
|
||||||
|
"echo 'swarm-payload' > /data/payload.txt",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"service",
|
||||||
|
"create",
|
||||||
|
"--name",
|
||||||
|
cls.service,
|
||||||
|
"--replicas",
|
||||||
|
"1",
|
||||||
|
"--mount",
|
||||||
|
f"type=volume,source={cls.volume},target=/data",
|
||||||
|
"alpine:3.20",
|
||||||
|
"sleep",
|
||||||
|
"3600",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
cls.task_cid = _task_container_id(cls.service)
|
||||||
|
cls.task_started_at = _started_at(cls.task_cid)
|
||||||
|
|
||||||
|
cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
|
||||||
|
write_databases_csv(cls.databases_csv, [])
|
||||||
|
|
||||||
|
# Whitelist that matches nothing: on main this forces a stop of every
|
||||||
|
# container at the volume, i.e. exactly the flake; on this branch the
|
||||||
|
# swarm task must be skipped instead. (An empty list would leave the
|
||||||
|
# --images-no-stop-required flag without arguments and argparse-fail.)
|
||||||
|
backup_run(
|
||||||
|
backups_dir=cls.backups_dir,
|
||||||
|
repo_name=cls.repo_name,
|
||||||
|
compose_dir=cls.compose_dir,
|
||||||
|
databases_csv=cls.databases_csv,
|
||||||
|
database_containers=["dummy-db"],
|
||||||
|
images_no_stop_required=["image-that-matches-nothing"],
|
||||||
|
)
|
||||||
|
cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
run(["docker", "service", "rm", cls.service], check=False)
|
||||||
|
deadline = time.time() + 30
|
||||||
|
while time.time() < deadline:
|
||||||
|
out = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"ps",
|
||||||
|
"-aq",
|
||||||
|
"--filter",
|
||||||
|
f"label=com.docker.swarm.service.name={cls.service}",
|
||||||
|
],
|
||||||
|
check=False,
|
||||||
|
).stdout.strip()
|
||||||
|
if not out:
|
||||||
|
break
|
||||||
|
time.sleep(2)
|
||||||
|
for v in cls.volumes:
|
||||||
|
run(["docker", "volume", "rm", "-f", v], check=False)
|
||||||
|
if cls.swarm_initted:
|
||||||
|
run(["docker", "swarm", "leave", "--force"], check=False)
|
||||||
|
|
||||||
|
def test_volume_backed_up_hot(self) -> None:
|
||||||
|
p = (
|
||||||
|
backup_path(
|
||||||
|
self.backups_dir,
|
||||||
|
self.repo_name,
|
||||||
|
self.version,
|
||||||
|
self.volume,
|
||||||
|
)
|
||||||
|
/ "files"
|
||||||
|
/ "payload.txt"
|
||||||
|
)
|
||||||
|
self.assertTrue(p.is_file(), f"Expected backed up file at: {p}")
|
||||||
|
|
||||||
|
def test_task_container_never_stopped(self) -> None:
|
||||||
|
out = run(
|
||||||
|
["docker", "ps", "-q", "--no-trunc", "--filter", f"id={self.task_cid}"]
|
||||||
|
).stdout.strip()
|
||||||
|
self.assertTrue(
|
||||||
|
out.startswith(self.task_cid) or self.task_cid.startswith(out.strip()[:12]),
|
||||||
|
f"Task container {self.task_cid} is no longer running",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
self.task_started_at,
|
||||||
|
_started_at(self.task_cid),
|
||||||
|
"Task container was restarted during the backup",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_service_never_replaced_the_task(self) -> None:
|
||||||
|
states = run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"service",
|
||||||
|
"ps",
|
||||||
|
self.service,
|
||||||
|
"--format",
|
||||||
|
"{{.DesiredState}} {{.CurrentState}}",
|
||||||
|
]
|
||||||
|
).stdout.strip()
|
||||||
|
lines = [line for line in states.splitlines() if line.strip()]
|
||||||
|
self.assertEqual(
|
||||||
|
len(lines), 1, f"Service task history shows replacements:\n{states}"
|
||||||
|
)
|
||||||
|
self.assertIn("Running", lines[0])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -6,8 +6,48 @@ import unittest
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
def run_seed(csv_path: Path, instance: str, database: str, username: str, password: str = "") -> subprocess.CompletedProcess:
|
def run_seed(
|
||||||
# Run the real CLI module (integration-style).
|
csv_path: Path, instance: str, database: str, username: str, password: str
|
||||||
|
) -> subprocess.CompletedProcess:
|
||||||
|
"""
|
||||||
|
Run the real CLI module (E2E-style) using subprocess.
|
||||||
|
|
||||||
|
Seed contract (current):
|
||||||
|
- database must be "*" or a valid name (non-empty, matches allowed charset)
|
||||||
|
- password is required
|
||||||
|
- entry is keyed by (instance, database); username/password get updated
|
||||||
|
"""
|
||||||
|
cp = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
"-m",
|
||||||
|
"baudolo.seed",
|
||||||
|
str(csv_path),
|
||||||
|
instance,
|
||||||
|
database,
|
||||||
|
username,
|
||||||
|
password,
|
||||||
|
],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if cp.returncode != 0:
|
||||||
|
raise AssertionError(
|
||||||
|
"seed command failed unexpectedly.\n"
|
||||||
|
f"returncode: {cp.returncode}\n"
|
||||||
|
f"stdout:\n{cp.stdout}\n"
|
||||||
|
f"stderr:\n{cp.stderr}\n"
|
||||||
|
)
|
||||||
|
return cp
|
||||||
|
|
||||||
|
|
||||||
|
def run_seed_expect_fail(
|
||||||
|
csv_path: Path, instance: str, database: str, username: str, password: str
|
||||||
|
) -> subprocess.CompletedProcess:
|
||||||
|
"""
|
||||||
|
Same as run_seed, but expects non-zero exit. Returns CompletedProcess for inspection.
|
||||||
|
"""
|
||||||
return subprocess.run(
|
return subprocess.run(
|
||||||
[
|
[
|
||||||
sys.executable,
|
sys.executable,
|
||||||
@@ -21,7 +61,7 @@ def run_seed(csv_path: Path, instance: str, database: str, username: str, passwo
|
|||||||
],
|
],
|
||||||
text=True,
|
text=True,
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
check=True,
|
check=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -31,6 +71,10 @@ def read_csv_semicolon(path: Path) -> list[dict]:
|
|||||||
return list(reader)
|
return list(reader)
|
||||||
|
|
||||||
|
|
||||||
|
def read_text(path: Path) -> str:
|
||||||
|
return path.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
class TestSeedIntegration(unittest.TestCase):
|
class TestSeedIntegration(unittest.TestCase):
|
||||||
def test_creates_file_and_adds_entry_when_missing(self) -> None:
|
def test_creates_file_and_adds_entry_when_missing(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as td:
|
with tempfile.TemporaryDirectory() as td:
|
||||||
@@ -39,7 +83,7 @@ class TestSeedIntegration(unittest.TestCase):
|
|||||||
|
|
||||||
cp = run_seed(p, "docker.test", "appdb", "alice", "secret")
|
cp = run_seed(p, "docker.test", "appdb", "alice", "secret")
|
||||||
|
|
||||||
self.assertEqual(cp.returncode, 0, cp.stderr)
|
self.assertEqual(cp.returncode, 0)
|
||||||
self.assertTrue(p.exists())
|
self.assertTrue(p.exists())
|
||||||
|
|
||||||
rows = read_csv_semicolon(p)
|
rows = read_csv_semicolon(p)
|
||||||
@@ -49,40 +93,121 @@ class TestSeedIntegration(unittest.TestCase):
|
|||||||
self.assertEqual(rows[0]["username"], "alice")
|
self.assertEqual(rows[0]["username"], "alice")
|
||||||
self.assertEqual(rows[0]["password"], "secret")
|
self.assertEqual(rows[0]["password"], "secret")
|
||||||
|
|
||||||
def test_replaces_existing_entry_same_keys(self) -> None:
|
def test_replaces_existing_entry_same_instance_and_database_updates_username_and_password(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Replacement semantics:
|
||||||
|
- Key is (instance, database)
|
||||||
|
- username/password are updated in-place
|
||||||
|
"""
|
||||||
with tempfile.TemporaryDirectory() as td:
|
with tempfile.TemporaryDirectory() as td:
|
||||||
p = Path(td) / "databases.csv"
|
p = Path(td) / "databases.csv"
|
||||||
|
|
||||||
# First add
|
|
||||||
run_seed(p, "docker.test", "appdb", "alice", "oldpw")
|
run_seed(p, "docker.test", "appdb", "alice", "oldpw")
|
||||||
rows = read_csv_semicolon(p)
|
rows = read_csv_semicolon(p)
|
||||||
self.assertEqual(len(rows), 1)
|
self.assertEqual(len(rows), 1)
|
||||||
|
self.assertEqual(rows[0]["username"], "alice")
|
||||||
self.assertEqual(rows[0]["password"], "oldpw")
|
self.assertEqual(rows[0]["password"], "oldpw")
|
||||||
|
|
||||||
# Replace (same instance+database+username)
|
run_seed(p, "docker.test", "appdb", "bob", "newpw")
|
||||||
run_seed(p, "docker.test", "appdb", "alice", "newpw")
|
|
||||||
rows = read_csv_semicolon(p)
|
rows = read_csv_semicolon(p)
|
||||||
|
|
||||||
self.assertEqual(len(rows), 1, "Expected replacement, not a duplicate row")
|
self.assertEqual(len(rows), 1, "Expected replacement, not a duplicate row")
|
||||||
self.assertEqual(rows[0]["instance"], "docker.test")
|
self.assertEqual(rows[0]["instance"], "docker.test")
|
||||||
self.assertEqual(rows[0]["database"], "appdb")
|
self.assertEqual(rows[0]["database"], "appdb")
|
||||||
self.assertEqual(rows[0]["username"], "alice")
|
self.assertEqual(rows[0]["username"], "bob")
|
||||||
self.assertEqual(rows[0]["password"], "newpw")
|
self.assertEqual(rows[0]["password"], "newpw")
|
||||||
|
|
||||||
def test_database_empty_string_matches_existing_empty_database(self) -> None:
|
def test_allows_star_database_for_dump_all(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as td:
|
with tempfile.TemporaryDirectory() as td:
|
||||||
p = Path(td) / "databases.csv"
|
p = Path(td) / "databases.csv"
|
||||||
|
|
||||||
# Add with empty database
|
cp = run_seed(p, "bigbluebutton", "*", "postgres", "pw")
|
||||||
run_seed(p, "docker.test", "", "alice", "pw1")
|
self.assertEqual(cp.returncode, 0)
|
||||||
|
|
||||||
rows = read_csv_semicolon(p)
|
rows = read_csv_semicolon(p)
|
||||||
self.assertEqual(len(rows), 1)
|
self.assertEqual(len(rows), 1)
|
||||||
self.assertEqual(rows[0]["database"], "")
|
self.assertEqual(rows[0]["instance"], "bigbluebutton")
|
||||||
|
self.assertEqual(rows[0]["database"], "*")
|
||||||
|
self.assertEqual(rows[0]["username"], "postgres")
|
||||||
|
self.assertEqual(rows[0]["password"], "pw")
|
||||||
|
|
||||||
|
def test_replaces_existing_star_entry(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
p = Path(td) / "databases.csv"
|
||||||
|
|
||||||
|
run_seed(p, "bigbluebutton", "*", "postgres", "pw1")
|
||||||
|
run_seed(p, "bigbluebutton", "*", "postgres", "pw2")
|
||||||
|
|
||||||
# Replace with empty database again
|
|
||||||
run_seed(p, "docker.test", "", "alice", "pw2")
|
|
||||||
rows = read_csv_semicolon(p)
|
rows = read_csv_semicolon(p)
|
||||||
|
|
||||||
self.assertEqual(len(rows), 1)
|
self.assertEqual(len(rows), 1)
|
||||||
self.assertEqual(rows[0]["database"], "")
|
self.assertEqual(rows[0]["database"], "*")
|
||||||
self.assertEqual(rows[0]["password"], "pw2")
|
self.assertEqual(rows[0]["password"], "pw2")
|
||||||
|
|
||||||
|
def test_rejects_empty_database_value(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
p = Path(td) / "databases.csv"
|
||||||
|
|
||||||
|
cp = run_seed_expect_fail(p, "docker.test", "", "alice", "pw")
|
||||||
|
self.assertNotEqual(cp.returncode, 0)
|
||||||
|
|
||||||
|
combined = ((cp.stdout or "") + "\n" + (cp.stderr or "")).lower()
|
||||||
|
self.assertIn("error:", combined)
|
||||||
|
self.assertIn("database", combined)
|
||||||
|
self.assertIn("not empty", combined)
|
||||||
|
|
||||||
|
self.assertFalse(p.exists(), "Should not create file on invalid input")
|
||||||
|
|
||||||
|
def test_rejects_invalid_database_name_characters(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
p = Path(td) / "databases.csv"
|
||||||
|
|
||||||
|
cp = run_seed_expect_fail(p, "docker.test", "app db", "alice", "pw")
|
||||||
|
self.assertNotEqual(cp.returncode, 0)
|
||||||
|
|
||||||
|
combined = ((cp.stdout or "") + "\n" + (cp.stderr or "")).lower()
|
||||||
|
self.assertIn("error:", combined)
|
||||||
|
self.assertIn("invalid database name", combined)
|
||||||
|
|
||||||
|
self.assertFalse(p.exists(), "Should not create file on invalid input")
|
||||||
|
|
||||||
|
def test_rejects_nan_database_name(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
p = Path(td) / "databases.csv"
|
||||||
|
|
||||||
|
cp = run_seed_expect_fail(p, "docker.test", "nan", "alice", "pw")
|
||||||
|
self.assertNotEqual(cp.returncode, 0)
|
||||||
|
|
||||||
|
combined = ((cp.stdout or "") + "\n" + (cp.stderr or "")).lower()
|
||||||
|
self.assertIn("error:", combined)
|
||||||
|
self.assertIn("must not be 'nan'", combined)
|
||||||
|
|
||||||
|
self.assertFalse(p.exists(), "Should not create file on invalid input")
|
||||||
|
|
||||||
|
def test_accepts_hyphen_and_underscore_database_names(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
p = Path(td) / "databases.csv"
|
||||||
|
|
||||||
|
run_seed(p, "docker.test", "my_db-1", "alice", "pw")
|
||||||
|
|
||||||
|
rows = read_csv_semicolon(p)
|
||||||
|
self.assertEqual(len(rows), 1)
|
||||||
|
self.assertEqual(rows[0]["database"], "my_db-1")
|
||||||
|
|
||||||
|
def test_file_is_semicolon_delimited_and_has_header(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
p = Path(td) / "databases.csv"
|
||||||
|
|
||||||
|
run_seed(p, "docker.test", "appdb", "alice", "pw")
|
||||||
|
|
||||||
|
txt = read_text(p)
|
||||||
|
self.assertTrue(
|
||||||
|
txt.startswith("instance;database;username;password"),
|
||||||
|
f"Unexpected header / delimiter in file:\n{txt}",
|
||||||
|
)
|
||||||
|
self.assertIn(";", txt)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
|
|||||||
0
tests/unit/backup/__init__.py
Normal file
0
tests/unit/backup/__init__.py
Normal file
77
tests/unit/backup/test_app_databases_csv.py
Normal file
77
tests/unit/backup/test_app_databases_csv.py
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
import io
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from contextlib import redirect_stderr
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
# Adjust if your package name/import path differs.
|
||||||
|
from baudolo.backup.app import _load_databases_df
|
||||||
|
|
||||||
|
|
||||||
|
EXPECTED_COLUMNS = ["instance", "database", "username", "password"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadDatabasesDf(unittest.TestCase):
|
||||||
|
def test_missing_csv_is_handled_with_warning_and_empty_df(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
missing_path = os.path.join(td, "does-not-exist.csv")
|
||||||
|
|
||||||
|
buf = io.StringIO()
|
||||||
|
with redirect_stderr(buf):
|
||||||
|
df = _load_databases_df(missing_path)
|
||||||
|
|
||||||
|
stderr = buf.getvalue()
|
||||||
|
self.assertIn("WARNING:", stderr)
|
||||||
|
self.assertIn("databases.csv not found", stderr)
|
||||||
|
|
||||||
|
self.assertIsInstance(df, pd.DataFrame)
|
||||||
|
self.assertListEqual(list(df.columns), EXPECTED_COLUMNS)
|
||||||
|
self.assertTrue(df.empty)
|
||||||
|
|
||||||
|
def test_empty_csv_is_handled_with_warning_and_empty_df(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
empty_path = os.path.join(td, "databases.csv")
|
||||||
|
# Create an empty file (0 bytes)
|
||||||
|
with open(empty_path, "w", encoding="utf-8") as f:
|
||||||
|
f.write("")
|
||||||
|
|
||||||
|
buf = io.StringIO()
|
||||||
|
with redirect_stderr(buf):
|
||||||
|
df = _load_databases_df(empty_path)
|
||||||
|
|
||||||
|
stderr = buf.getvalue()
|
||||||
|
self.assertIn("WARNING:", stderr)
|
||||||
|
self.assertIn("exists but is empty", stderr)
|
||||||
|
|
||||||
|
self.assertIsInstance(df, pd.DataFrame)
|
||||||
|
self.assertListEqual(list(df.columns), EXPECTED_COLUMNS)
|
||||||
|
self.assertTrue(df.empty)
|
||||||
|
|
||||||
|
def test_valid_csv_loads_without_warning(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
csv_path = os.path.join(td, "databases.csv")
|
||||||
|
|
||||||
|
content = "instance;database;username;password\nmyapp;*;dbuser;secret\n"
|
||||||
|
with open(csv_path, "w", encoding="utf-8") as f:
|
||||||
|
f.write(content)
|
||||||
|
|
||||||
|
buf = io.StringIO()
|
||||||
|
with redirect_stderr(buf):
|
||||||
|
df = _load_databases_df(csv_path)
|
||||||
|
|
||||||
|
stderr = buf.getvalue()
|
||||||
|
self.assertEqual(stderr, "") # no warning expected
|
||||||
|
|
||||||
|
self.assertIsInstance(df, pd.DataFrame)
|
||||||
|
self.assertListEqual(list(df.columns), EXPECTED_COLUMNS)
|
||||||
|
self.assertEqual(len(df), 1)
|
||||||
|
self.assertEqual(df.loc[0, "instance"], "myapp")
|
||||||
|
self.assertEqual(df.loc[0, "database"], "*")
|
||||||
|
self.assertEqual(df.loc[0, "username"], "dbuser")
|
||||||
|
self.assertEqual(df.loc[0, "password"], "secret")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
255
tests/unit/backup/test_compose.py
Normal file
255
tests/unit/backup/test_compose.py
Normal file
@@ -0,0 +1,255 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
|
def _touch(p: Path) -> None:
|
||||||
|
p.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# If the path already exists as a directory (e.g. ".env" created by ".env/env"),
|
||||||
|
# remove it so we can create a file with the same name.
|
||||||
|
if p.exists() and p.is_dir():
|
||||||
|
shutil.rmtree(p)
|
||||||
|
|
||||||
|
p.write_text("x", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _setup_compose_dir(
|
||||||
|
tmp_path: Path,
|
||||||
|
name: str = "mailu",
|
||||||
|
*,
|
||||||
|
compose_name: str = "docker-compose.yml",
|
||||||
|
with_override: bool = False,
|
||||||
|
with_ca_override: bool = False,
|
||||||
|
env_layout: str | None = None, # None | ".env" | ".env/env"
|
||||||
|
) -> Path:
|
||||||
|
d = tmp_path / name
|
||||||
|
d.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
_touch(d / compose_name)
|
||||||
|
|
||||||
|
if with_override:
|
||||||
|
_touch(d / "docker-compose.override.yml")
|
||||||
|
|
||||||
|
if with_ca_override:
|
||||||
|
_touch(d / "docker-compose.ca.override.yml")
|
||||||
|
|
||||||
|
if env_layout == ".env":
|
||||||
|
_touch(d / ".env")
|
||||||
|
elif env_layout == ".env/env":
|
||||||
|
_touch(d / ".env" / "env")
|
||||||
|
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
class TestCompose(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
from baudolo.backup import compose as mod
|
||||||
|
|
||||||
|
cls.compose_mod = mod
|
||||||
|
|
||||||
|
def test_find_compose_file_supports_all_valid_names_case_insensitive(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
tmp_path = Path(td)
|
||||||
|
|
||||||
|
variants = [
|
||||||
|
"compose.yml",
|
||||||
|
"compose.yaml",
|
||||||
|
"docker-compose.yml",
|
||||||
|
"docker-compose.yaml",
|
||||||
|
"docker-compose.yAml",
|
||||||
|
]
|
||||||
|
|
||||||
|
for i, name in enumerate(variants):
|
||||||
|
d = _setup_compose_dir(
|
||||||
|
tmp_path,
|
||||||
|
name=f"project{i}",
|
||||||
|
compose_name=name,
|
||||||
|
)
|
||||||
|
found = self.compose_mod._find_compose_file(str(d))
|
||||||
|
self.assertIsNotNone(found)
|
||||||
|
self.assertEqual(found.name, name)
|
||||||
|
|
||||||
|
def test_find_compose_file_returns_none_when_missing(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
tmp_path = Path(td)
|
||||||
|
d = tmp_path / "empty"
|
||||||
|
d.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
found = self.compose_mod._find_compose_file(str(d))
|
||||||
|
self.assertIsNone(found)
|
||||||
|
|
||||||
|
def test_build_cmd_uses_wrapper_when_present(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
tmp_path = Path(td)
|
||||||
|
d = _setup_compose_dir(
|
||||||
|
tmp_path,
|
||||||
|
with_override=True,
|
||||||
|
with_ca_override=True,
|
||||||
|
env_layout=".env",
|
||||||
|
)
|
||||||
|
|
||||||
|
def fake_which(name: str):
|
||||||
|
if name == "compose":
|
||||||
|
return "/usr/local/bin/compose"
|
||||||
|
return None
|
||||||
|
|
||||||
|
with patch.object(self.compose_mod.shutil, "which", fake_which):
|
||||||
|
cmd = self.compose_mod._build_compose_cmd(str(d), ["up", "-d"])
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
cmd,
|
||||||
|
[
|
||||||
|
"/usr/local/bin/compose",
|
||||||
|
"--chdir",
|
||||||
|
str(d.resolve()),
|
||||||
|
"--",
|
||||||
|
"up",
|
||||||
|
"-d",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_build_cmd_fallback_uses_plain_docker_compose_chdir(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
tmp_path = Path(td)
|
||||||
|
d = _setup_compose_dir(
|
||||||
|
tmp_path,
|
||||||
|
with_override=True,
|
||||||
|
with_ca_override=True,
|
||||||
|
env_layout=".env",
|
||||||
|
)
|
||||||
|
|
||||||
|
def fake_which(name: str):
|
||||||
|
if name == "compose":
|
||||||
|
return None
|
||||||
|
if name == "docker":
|
||||||
|
return "/usr/bin/docker"
|
||||||
|
return None
|
||||||
|
|
||||||
|
with patch.object(self.compose_mod.shutil, "which", fake_which):
|
||||||
|
cmd = self.compose_mod._build_compose_cmd(
|
||||||
|
str(d), ["up", "-d", "--force-recreate"]
|
||||||
|
)
|
||||||
|
|
||||||
|
expected: List[str] = [
|
||||||
|
"/usr/bin/docker",
|
||||||
|
"compose",
|
||||||
|
"--chdir",
|
||||||
|
str(d.resolve()),
|
||||||
|
"up",
|
||||||
|
"-d",
|
||||||
|
"--force-recreate",
|
||||||
|
]
|
||||||
|
self.assertEqual(cmd, expected)
|
||||||
|
|
||||||
|
def test_hard_restart_calls_run_twice_with_correct_cmds_wrapper(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
tmp_path = Path(td)
|
||||||
|
d = _setup_compose_dir(tmp_path, name="mailu", env_layout=".env")
|
||||||
|
|
||||||
|
def fake_which(name: str):
|
||||||
|
if name == "compose":
|
||||||
|
return "/usr/local/bin/compose"
|
||||||
|
return None
|
||||||
|
|
||||||
|
with patch.object(self.compose_mod.shutil, "which", fake_which):
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake_run(cmd, check: bool):
|
||||||
|
calls.append((cmd, check))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with patch.object(self.compose_mod.subprocess, "run", fake_run):
|
||||||
|
self.compose_mod.hard_restart_docker_services(str(d))
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
calls,
|
||||||
|
[
|
||||||
|
(
|
||||||
|
[
|
||||||
|
"/usr/local/bin/compose",
|
||||||
|
"--chdir",
|
||||||
|
str(d.resolve()),
|
||||||
|
"--",
|
||||||
|
"down",
|
||||||
|
],
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
[
|
||||||
|
"/usr/local/bin/compose",
|
||||||
|
"--chdir",
|
||||||
|
str(d.resolve()),
|
||||||
|
"--",
|
||||||
|
"up",
|
||||||
|
"-d",
|
||||||
|
],
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_hard_restart_calls_run_twice_with_correct_cmds_fallback(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
tmp_path = Path(td)
|
||||||
|
d = _setup_compose_dir(
|
||||||
|
tmp_path,
|
||||||
|
name="mailu",
|
||||||
|
with_override=True,
|
||||||
|
with_ca_override=True,
|
||||||
|
env_layout=".env/env",
|
||||||
|
)
|
||||||
|
|
||||||
|
def fake_which(name: str):
|
||||||
|
if name == "compose":
|
||||||
|
return None
|
||||||
|
if name == "docker":
|
||||||
|
return "/usr/bin/docker"
|
||||||
|
return None
|
||||||
|
|
||||||
|
with patch.object(self.compose_mod.shutil, "which", fake_which):
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake_run(cmd, check: bool):
|
||||||
|
calls.append((cmd, check))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with patch.object(self.compose_mod.subprocess, "run", fake_run):
|
||||||
|
self.compose_mod.hard_restart_docker_services(str(d))
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
calls,
|
||||||
|
[
|
||||||
|
(
|
||||||
|
[
|
||||||
|
"/usr/bin/docker",
|
||||||
|
"compose",
|
||||||
|
"--chdir",
|
||||||
|
str(d.resolve()),
|
||||||
|
"down",
|
||||||
|
],
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
[
|
||||||
|
"/usr/bin/docker",
|
||||||
|
"compose",
|
||||||
|
"--chdir",
|
||||||
|
str(d.resolve()),
|
||||||
|
"up",
|
||||||
|
"-d",
|
||||||
|
],
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
70
tests/unit/backup/test_db_mariadb_dump.py
Normal file
70
tests/unit/backup/test_db_mariadb_dump.py
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pandas
|
||||||
|
|
||||||
|
from baudolo.backup import db as db_mod
|
||||||
|
|
||||||
|
|
||||||
|
def _df(rows):
|
||||||
|
return pandas.DataFrame(
|
||||||
|
rows, columns=["instance", "database", "username", "password"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _capture_commands(*, db_type, rows, container):
|
||||||
|
captured = []
|
||||||
|
|
||||||
|
def _capture(cmd):
|
||||||
|
captured.append(cmd)
|
||||||
|
return []
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
with patch.object(db_mod, "execute_shell_command", side_effect=_capture):
|
||||||
|
db_mod.backup_database(
|
||||||
|
container=container,
|
||||||
|
volume_dir=td,
|
||||||
|
db_type=db_type,
|
||||||
|
databases_df=_df(rows),
|
||||||
|
database_containers=[container],
|
||||||
|
)
|
||||||
|
return captured
|
||||||
|
|
||||||
|
|
||||||
|
class TestMariaDBDumpUsesTCP(unittest.TestCase):
|
||||||
|
# Regression guard for 'Access denied for user <user>@localhost' when only
|
||||||
|
# '<user>'@'%' is granted: the in-container mariadb-dump MUST force TCP so
|
||||||
|
# the connection is auth-matched against '%' instead of socket->localhost.
|
||||||
|
|
||||||
|
def test_mariadb_dump_forces_tcp_loopback(self):
|
||||||
|
captured = _capture_commands(
|
||||||
|
db_type="mariadb",
|
||||||
|
rows=[("mariadb", "appdb", "appuser", "s3cret")],
|
||||||
|
container="mariadb",
|
||||||
|
)
|
||||||
|
dump_cmds = [c for c in captured if "mariadb-dump" in c]
|
||||||
|
self.assertEqual(
|
||||||
|
len(dump_cmds), 1, f"expected one dump command, got: {captured}"
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd = dump_cmds[0]
|
||||||
|
self.assertIn("-h 127.0.0.1", cmd)
|
||||||
|
self.assertIn("--protocol=tcp", cmd)
|
||||||
|
self.assertIn("-u appuser", cmd)
|
||||||
|
self.assertIn("-ps3cret", cmd)
|
||||||
|
self.assertIn(" appdb", cmd)
|
||||||
|
|
||||||
|
def test_postgres_dump_unaffected(self):
|
||||||
|
captured = _capture_commands(
|
||||||
|
db_type="postgres",
|
||||||
|
rows=[("pg", "appdb", "appuser", "s3cret")],
|
||||||
|
container="pg",
|
||||||
|
)
|
||||||
|
dump_cmds = [c for c in captured if "pg_dump" in c and "pg_dumpall" not in c]
|
||||||
|
self.assertEqual(len(dump_cmds), 1)
|
||||||
|
self.assertNotIn("--protocol=tcp", dump_cmds[0])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
53
tests/unit/backup/test_docker_swarm.py
Normal file
53
tests/unit/backup/test_docker_swarm.py
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from baudolo.backup import docker as docker_mod
|
||||||
|
from baudolo.backup.shell import BackupException
|
||||||
|
|
||||||
|
|
||||||
|
class TestIsSwarmTask(unittest.TestCase):
|
||||||
|
@patch.object(docker_mod, "execute_shell_command", return_value=["task-id-123"])
|
||||||
|
def test_true_when_task_label_present(self, _mock) -> None:
|
||||||
|
self.assertTrue(docker_mod.is_swarm_task("c1"))
|
||||||
|
|
||||||
|
@patch.object(docker_mod, "execute_shell_command", return_value=[""])
|
||||||
|
def test_false_when_label_empty(self, _mock) -> None:
|
||||||
|
self.assertFalse(docker_mod.is_swarm_task("c1"))
|
||||||
|
|
||||||
|
@patch.object(docker_mod, "execute_shell_command", return_value=[])
|
||||||
|
def test_false_when_no_output(self, _mock) -> None:
|
||||||
|
self.assertFalse(docker_mod.is_swarm_task("c1"))
|
||||||
|
|
||||||
|
@patch.object(
|
||||||
|
docker_mod,
|
||||||
|
"execute_shell_command",
|
||||||
|
side_effect=[BackupException("gone"), []],
|
||||||
|
)
|
||||||
|
def test_vanished_container_counts_as_not_stoppable(self, _mock) -> None:
|
||||||
|
# A container removed between listing and inspect must not abort the
|
||||||
|
# whole backup run; treating it as a swarm task keeps it out of every
|
||||||
|
# stop/start and image-inspect path.
|
||||||
|
self.assertTrue(docker_mod.is_swarm_task("gone-container"))
|
||||||
|
|
||||||
|
@patch.object(
|
||||||
|
docker_mod,
|
||||||
|
"execute_shell_command",
|
||||||
|
side_effect=[BackupException("daemon hiccup"), ["still-here"]],
|
||||||
|
)
|
||||||
|
def test_inspect_failure_on_existing_container_still_fails(self, _mock) -> None:
|
||||||
|
# If the container still exists, an inspect failure must keep failing
|
||||||
|
# the run: silently skipping the stop would back up a hot volume and
|
||||||
|
# report green without the stop guarantee.
|
||||||
|
with self.assertRaises(BackupException):
|
||||||
|
docker_mod.is_swarm_task("still-here")
|
||||||
|
|
||||||
|
|
||||||
|
class TestFilterStoppable(unittest.TestCase):
|
||||||
|
@patch.object(docker_mod, "is_swarm_task", side_effect=[False, True, False])
|
||||||
|
def test_mixed_list_keeps_order_and_drops_tasks(self, _mock) -> None:
|
||||||
|
result = docker_mod.filter_stoppable(["plain-1", "swarm-task", "plain-2"])
|
||||||
|
self.assertEqual(result, ["plain-1", "plain-2"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
0
tests/unit/restore/__init__.py
Normal file
0
tests/unit/restore/__init__.py
Normal file
44
tests/unit/restore/test_mariadb_empty_drop.py
Normal file
44
tests/unit/restore/test_mariadb_empty_drop.py
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from baudolo.restore.db import mariadb as mariadb_mod
|
||||||
|
|
||||||
|
|
||||||
|
class TestMariadbEmptyDrop(unittest.TestCase):
|
||||||
|
def test_drops_run_in_one_session_with_fk_checks_off(self) -> None:
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def _capture(container, argv, **kwargs):
|
||||||
|
calls.append(argv)
|
||||||
|
result = MagicMock()
|
||||||
|
result.stdout = b"users\nfetches\n"
|
||||||
|
return result
|
||||||
|
|
||||||
|
with tempfile.NamedTemporaryFile(suffix=".sql") as sql:
|
||||||
|
sql.write(b"CREATE TABLE t (id int);\n")
|
||||||
|
sql.flush()
|
||||||
|
with (
|
||||||
|
patch.object(mariadb_mod, "docker_exec", side_effect=_capture),
|
||||||
|
patch.object(mariadb_mod, "_pick_client", return_value="mariadb"),
|
||||||
|
):
|
||||||
|
mariadb_mod.restore_mariadb_sql(
|
||||||
|
container="db",
|
||||||
|
db_name="mailu",
|
||||||
|
user="mailu",
|
||||||
|
password="pw",
|
||||||
|
sql_path=sql.name,
|
||||||
|
empty=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
drop_calls = [argv for argv in calls if any("DROP TABLE" in a for a in argv)]
|
||||||
|
self.assertEqual(len(drop_calls), 1, f"expected ONE drop session: {calls}")
|
||||||
|
drop_sql = drop_calls[0][-1]
|
||||||
|
self.assertTrue(drop_sql.startswith("SET FOREIGN_KEY_CHECKS=0; "))
|
||||||
|
self.assertIn("DROP TABLE IF EXISTS `mailu`.`users`;", drop_sql)
|
||||||
|
self.assertIn("DROP TABLE IF EXISTS `mailu`.`fetches`;", drop_sql)
|
||||||
|
self.assertTrue(drop_sql.endswith("SET FOREIGN_KEY_CHECKS=1;"))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
43
tests/unit/restore/test_postgres_empty_drop.py
Normal file
43
tests/unit/restore/test_postgres_empty_drop.py
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
"""The --empty pre-clean drop must cover every object class a dump can
|
||||||
|
re-CREATE ahead of tables; a missed class aborts the replay under
|
||||||
|
ON_ERROR_STOP (OpenProject's ICU collation public.versions_name did)."""
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from baudolo.restore.db import postgres
|
||||||
|
|
||||||
|
|
||||||
|
class TestPostgresEmptyDrop(unittest.TestCase):
|
||||||
|
def _drop_sql(self) -> str:
|
||||||
|
with tempfile.NamedTemporaryFile(suffix=".sql") as fh:
|
||||||
|
Path(fh.name).write_bytes(b"SELECT 1;\n")
|
||||||
|
with mock.patch.object(postgres, "docker_exec") as run:
|
||||||
|
postgres.restore_postgres_sql(
|
||||||
|
container="c",
|
||||||
|
db_name="db",
|
||||||
|
user="u",
|
||||||
|
password="p",
|
||||||
|
sql_path=fh.name,
|
||||||
|
empty=True,
|
||||||
|
)
|
||||||
|
first_call = run.call_args_list[0]
|
||||||
|
return first_call.kwargs["stdin"].decode()
|
||||||
|
|
||||||
|
def test_drop_covers_collations(self) -> None:
|
||||||
|
sql = self._drop_sql()
|
||||||
|
self.assertIn("pg_collation", sql)
|
||||||
|
self.assertIn("'COLLATION' AS type", sql)
|
||||||
|
self.assertIn("pg_get_userbyid(col.collowner) = current_user", sql)
|
||||||
|
|
||||||
|
def test_drop_still_covers_the_other_classes(self) -> None:
|
||||||
|
sql = self._drop_sql()
|
||||||
|
for marker in ("pg_class", "pg_proc", "'SEQUENCE' AS type", "'TYPE' AS type"):
|
||||||
|
self.assertIn(marker, sql)
|
||||||
|
self.assertIn("DROP %s IF EXISTS public.%I CASCADE", sql)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
63
tests/unit/restore/test_postgres_filter.py
Normal file
63
tests/unit/restore/test_postgres_filter.py
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from baudolo.restore.db.postgres import filter_superuser_only_lines
|
||||||
|
|
||||||
|
|
||||||
|
def _filter(raw: bytes) -> bytes:
|
||||||
|
return b"".join(filter_superuser_only_lines(raw.splitlines(keepends=True)))
|
||||||
|
|
||||||
|
|
||||||
|
class TestFilterSuperuserOnlyLines(unittest.TestCase):
|
||||||
|
def test_drops_superuser_only_statements(self) -> None:
|
||||||
|
raw = (
|
||||||
|
b"CREATE TABLE t (id int);\n"
|
||||||
|
b"COMMENT ON EXTENSION pg_trgm IS 'trigram';\n"
|
||||||
|
b"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO x;\n"
|
||||||
|
b"INSERT INTO t VALUES (1);\n"
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
_filter(raw),
|
||||||
|
b"CREATE TABLE t (id int);\nINSERT INTO t VALUES (1);\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_copy_data_rows_are_never_filtered(self) -> None:
|
||||||
|
raw = (
|
||||||
|
b"COPY public.snippets (body) FROM stdin;\n"
|
||||||
|
b"COMMENT ON EXTENSION looks like sql but is data\n"
|
||||||
|
b"ALTER DEFAULT PRIVILEGES stored as text\n"
|
||||||
|
b"\\.\n"
|
||||||
|
b"ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON TABLES FROM y;\n"
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
_filter(raw),
|
||||||
|
b"COPY public.snippets (body) FROM stdin;\n"
|
||||||
|
b"COMMENT ON EXTENSION looks like sql but is data\n"
|
||||||
|
b"ALTER DEFAULT PRIVILEGES stored as text\n"
|
||||||
|
b"\\.\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_consecutive_copy_blocks_keep_state(self) -> None:
|
||||||
|
raw = (
|
||||||
|
b"COPY public.a (v) FROM stdin;\n"
|
||||||
|
b"row-a\n"
|
||||||
|
b"\\.\n"
|
||||||
|
b"COMMENT ON EXTENSION dropme IS 'x';\n"
|
||||||
|
b"COPY public.b (v) FROM stdin;\n"
|
||||||
|
b"COMMENT ON EXTENSION kept-as-data\n"
|
||||||
|
b"\\.\n"
|
||||||
|
)
|
||||||
|
out = _filter(raw)
|
||||||
|
self.assertNotIn(b"dropme", out)
|
||||||
|
self.assertIn(b"COMMENT ON EXTENSION kept-as-data\n", out)
|
||||||
|
|
||||||
|
def test_everything_else_passes_through_verbatim(self) -> None:
|
||||||
|
raw = (
|
||||||
|
b"SET statement_timeout = 0;\n"
|
||||||
|
b"CREATE EXTENSION IF NOT EXISTS pg_trgm;\n"
|
||||||
|
b"GRANT ALL ON SCHEMA public TO app;\n"
|
||||||
|
)
|
||||||
|
self.assertEqual(_filter(raw), raw)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
0
tests/unit/seed/__init__.py
Normal file
0
tests/unit/seed/__init__.py
Normal file
215
tests/unit/seed/test_main.py
Normal file
215
tests/unit/seed/test_main.py
Normal file
@@ -0,0 +1,215 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
from pandas.errors import EmptyDataError
|
||||||
|
|
||||||
|
import baudolo.seed.__main__ as seed_main
|
||||||
|
|
||||||
|
|
||||||
|
class TestSeedMain(unittest.TestCase):
|
||||||
|
@patch("baudolo.seed.__main__.pd.DataFrame")
|
||||||
|
def test_empty_df_creates_expected_columns(self, df_ctor: MagicMock) -> None:
|
||||||
|
seed_main._empty_df()
|
||||||
|
df_ctor.assert_called_once_with(
|
||||||
|
columns=["instance", "database", "username", "password"]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_validate_database_value_rejects_empty(self) -> None:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
seed_main._validate_database_value("", instance="x")
|
||||||
|
|
||||||
|
def test_validate_database_value_accepts_star(self) -> None:
|
||||||
|
self.assertEqual(seed_main._validate_database_value("*", instance="x"), "*")
|
||||||
|
|
||||||
|
def test_validate_database_value_rejects_nan(self) -> None:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
seed_main._validate_database_value("nan", instance="x")
|
||||||
|
|
||||||
|
def test_validate_database_value_rejects_invalid_name(self) -> None:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
seed_main._validate_database_value("bad name", instance="x")
|
||||||
|
|
||||||
|
def _mock_df_mask_any(self, *, any_value: bool) -> MagicMock:
|
||||||
|
"""
|
||||||
|
Build a DataFrame-like mock such that:
|
||||||
|
mask = (df["instance"] == instance) & (df["database"] == database)
|
||||||
|
mask.any() returns any_value
|
||||||
|
"""
|
||||||
|
df = MagicMock(spec=pd.DataFrame)
|
||||||
|
|
||||||
|
left = MagicMock()
|
||||||
|
right = MagicMock()
|
||||||
|
|
||||||
|
mask = MagicMock()
|
||||||
|
mask.any.return_value = any_value
|
||||||
|
|
||||||
|
# (left & right) => mask
|
||||||
|
left.__and__.return_value = mask
|
||||||
|
|
||||||
|
# df["instance"] / df["database"] => return objects whose == produces left/right
|
||||||
|
col = MagicMock()
|
||||||
|
col.__eq__.side_effect = [left, right]
|
||||||
|
df.__getitem__.return_value = col
|
||||||
|
|
||||||
|
return df
|
||||||
|
|
||||||
|
@patch("baudolo.seed.__main__.os.path.exists", return_value=False)
|
||||||
|
@patch("baudolo.seed.__main__.pd.read_csv")
|
||||||
|
@patch("baudolo.seed.__main__._empty_df")
|
||||||
|
@patch("baudolo.seed.__main__.pd.concat")
|
||||||
|
def test_check_and_add_entry_file_missing_adds_entry(
|
||||||
|
self,
|
||||||
|
concat: MagicMock,
|
||||||
|
empty_df: MagicMock,
|
||||||
|
read_csv: MagicMock,
|
||||||
|
exists: MagicMock,
|
||||||
|
) -> None:
|
||||||
|
df_existing = self._mock_df_mask_any(any_value=False)
|
||||||
|
empty_df.return_value = df_existing
|
||||||
|
|
||||||
|
df_out = MagicMock(spec=pd.DataFrame)
|
||||||
|
concat.return_value = df_out
|
||||||
|
|
||||||
|
seed_main.check_and_add_entry(
|
||||||
|
file_path="/tmp/databases.csv",
|
||||||
|
instance="inst",
|
||||||
|
database="db",
|
||||||
|
username="user",
|
||||||
|
password="pass",
|
||||||
|
)
|
||||||
|
|
||||||
|
read_csv.assert_not_called()
|
||||||
|
empty_df.assert_called_once()
|
||||||
|
concat.assert_called_once()
|
||||||
|
df_out.to_csv.assert_called_once_with(
|
||||||
|
"/tmp/databases.csv", sep=";", index=False
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("baudolo.seed.__main__.os.path.exists", return_value=True)
|
||||||
|
@patch("baudolo.seed.__main__.pd.read_csv", side_effect=EmptyDataError("empty"))
|
||||||
|
@patch("baudolo.seed.__main__._empty_df")
|
||||||
|
@patch("baudolo.seed.__main__.pd.concat")
|
||||||
|
@patch("baudolo.seed.__main__.print")
|
||||||
|
def test_check_and_add_entry_empty_file_warns_and_creates_columns_and_adds(
|
||||||
|
self,
|
||||||
|
print_: MagicMock,
|
||||||
|
concat: MagicMock,
|
||||||
|
empty_df: MagicMock,
|
||||||
|
read_csv: MagicMock,
|
||||||
|
exists: MagicMock,
|
||||||
|
) -> None:
|
||||||
|
df_existing = self._mock_df_mask_any(any_value=False)
|
||||||
|
empty_df.return_value = df_existing
|
||||||
|
|
||||||
|
df_out = MagicMock(spec=pd.DataFrame)
|
||||||
|
concat.return_value = df_out
|
||||||
|
|
||||||
|
seed_main.check_and_add_entry(
|
||||||
|
file_path="/tmp/databases.csv",
|
||||||
|
instance="inst",
|
||||||
|
database="db",
|
||||||
|
username="user",
|
||||||
|
password="pass",
|
||||||
|
)
|
||||||
|
|
||||||
|
exists.assert_called_once_with("/tmp/databases.csv")
|
||||||
|
read_csv.assert_called_once()
|
||||||
|
empty_df.assert_called_once()
|
||||||
|
concat.assert_called_once()
|
||||||
|
|
||||||
|
# Assert: at least one print call contains the WARNING and prints to stderr
|
||||||
|
warning_calls = []
|
||||||
|
for call in print_.call_args_list:
|
||||||
|
args, kwargs = call
|
||||||
|
if args and "WARNING: databases.csv exists but is empty" in str(args[0]):
|
||||||
|
warning_calls.append((args, kwargs))
|
||||||
|
|
||||||
|
self.assertTrue(
|
||||||
|
warning_calls,
|
||||||
|
"Expected a WARNING print when databases.csv is empty, but none was found.",
|
||||||
|
)
|
||||||
|
# Ensure the warning goes to stderr
|
||||||
|
_, warn_kwargs = warning_calls[0]
|
||||||
|
self.assertEqual(warn_kwargs.get("file"), seed_main.sys.stderr)
|
||||||
|
|
||||||
|
df_out.to_csv.assert_called_once_with(
|
||||||
|
"/tmp/databases.csv", sep=";", index=False
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("baudolo.seed.__main__.os.path.exists", return_value=True)
|
||||||
|
@patch("baudolo.seed.__main__.pd.read_csv")
|
||||||
|
def test_check_and_add_entry_updates_existing_row(
|
||||||
|
self,
|
||||||
|
read_csv: MagicMock,
|
||||||
|
exists: MagicMock,
|
||||||
|
) -> None:
|
||||||
|
df = self._mock_df_mask_any(any_value=True)
|
||||||
|
read_csv.return_value = df
|
||||||
|
|
||||||
|
seed_main.check_and_add_entry(
|
||||||
|
file_path="/tmp/databases.csv",
|
||||||
|
instance="inst",
|
||||||
|
database="db",
|
||||||
|
username="user",
|
||||||
|
password="pass",
|
||||||
|
)
|
||||||
|
|
||||||
|
df.to_csv.assert_called_once_with("/tmp/databases.csv", sep=";", index=False)
|
||||||
|
|
||||||
|
@patch("baudolo.seed.__main__.check_and_add_entry")
|
||||||
|
@patch("baudolo.seed.__main__.argparse.ArgumentParser.parse_args")
|
||||||
|
def test_main_calls_check_and_add_entry(
|
||||||
|
self, parse_args: MagicMock, cae: MagicMock
|
||||||
|
) -> None:
|
||||||
|
ns = MagicMock()
|
||||||
|
ns.file = "/tmp/databases.csv"
|
||||||
|
ns.instance = "inst"
|
||||||
|
ns.database = "db"
|
||||||
|
ns.username = "user"
|
||||||
|
ns.password = "pass"
|
||||||
|
parse_args.return_value = ns
|
||||||
|
|
||||||
|
seed_main.main()
|
||||||
|
|
||||||
|
cae.assert_called_once_with(
|
||||||
|
file_path="/tmp/databases.csv",
|
||||||
|
instance="inst",
|
||||||
|
database="db",
|
||||||
|
username="user",
|
||||||
|
password="pass",
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("baudolo.seed.__main__.sys.exit")
|
||||||
|
@patch("baudolo.seed.__main__.print")
|
||||||
|
@patch(
|
||||||
|
"baudolo.seed.__main__.check_and_add_entry", side_effect=RuntimeError("boom")
|
||||||
|
)
|
||||||
|
@patch("baudolo.seed.__main__.argparse.ArgumentParser.parse_args")
|
||||||
|
def test_main_exits_nonzero_on_error(
|
||||||
|
self,
|
||||||
|
parse_args: MagicMock,
|
||||||
|
cae: MagicMock,
|
||||||
|
print_: MagicMock,
|
||||||
|
exit_: MagicMock,
|
||||||
|
) -> None:
|
||||||
|
ns = MagicMock()
|
||||||
|
ns.file = "/tmp/databases.csv"
|
||||||
|
ns.instance = "inst"
|
||||||
|
ns.database = "db"
|
||||||
|
ns.username = "user"
|
||||||
|
ns.password = "pass"
|
||||||
|
parse_args.return_value = ns
|
||||||
|
|
||||||
|
seed_main.main()
|
||||||
|
|
||||||
|
self.assertTrue(print_.called)
|
||||||
|
_, kwargs = print_.call_args
|
||||||
|
self.assertEqual(kwargs.get("file"), seed_main.sys.stderr)
|
||||||
|
exit_.assert_called_once_with(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
@@ -4,9 +4,12 @@ from unittest.mock import patch
|
|||||||
from baudolo.backup.app import requires_stop
|
from baudolo.backup.app import requires_stop
|
||||||
|
|
||||||
|
|
||||||
|
@patch("baudolo.backup.app.is_swarm_task", return_value=False)
|
||||||
class TestRequiresStop(unittest.TestCase):
|
class TestRequiresStop(unittest.TestCase):
|
||||||
@patch("baudolo.backup.app.get_image_info")
|
@patch("baudolo.backup.app.get_image_info")
|
||||||
def test_requires_stop_false_when_all_images_are_whitelisted(self, mock_get_image_info):
|
def test_requires_stop_false_when_all_images_are_whitelisted(
|
||||||
|
self, mock_get_image_info, _mock_is_swarm_task
|
||||||
|
):
|
||||||
# All containers use images containing allowed substrings
|
# All containers use images containing allowed substrings
|
||||||
mock_get_image_info.side_effect = [
|
mock_get_image_info.side_effect = [
|
||||||
"repo/mastodon:v4",
|
"repo/mastodon:v4",
|
||||||
@@ -17,7 +20,9 @@ class TestRequiresStop(unittest.TestCase):
|
|||||||
self.assertFalse(requires_stop(containers, whitelist))
|
self.assertFalse(requires_stop(containers, whitelist))
|
||||||
|
|
||||||
@patch("baudolo.backup.app.get_image_info")
|
@patch("baudolo.backup.app.get_image_info")
|
||||||
def test_requires_stop_true_when_any_image_is_not_whitelisted(self, mock_get_image_info):
|
def test_requires_stop_true_when_any_image_is_not_whitelisted(
|
||||||
|
self, mock_get_image_info, _mock_is_swarm_task
|
||||||
|
):
|
||||||
mock_get_image_info.side_effect = [
|
mock_get_image_info.side_effect = [
|
||||||
"repo/mastodon:v4",
|
"repo/mastodon:v4",
|
||||||
"repo/nginx:latest",
|
"repo/nginx:latest",
|
||||||
@@ -27,10 +32,23 @@ class TestRequiresStop(unittest.TestCase):
|
|||||||
self.assertTrue(requires_stop(containers, whitelist))
|
self.assertTrue(requires_stop(containers, whitelist))
|
||||||
|
|
||||||
@patch("baudolo.backup.app.get_image_info")
|
@patch("baudolo.backup.app.get_image_info")
|
||||||
def test_requires_stop_true_when_whitelist_empty(self, mock_get_image_info):
|
def test_requires_stop_true_when_whitelist_empty(
|
||||||
|
self, mock_get_image_info, _mock_is_swarm_task
|
||||||
|
):
|
||||||
mock_get_image_info.return_value = "repo/anything:latest"
|
mock_get_image_info.return_value = "repo/anything:latest"
|
||||||
self.assertTrue(requires_stop(["c1"], []))
|
self.assertTrue(requires_stop(["c1"], []))
|
||||||
|
|
||||||
|
|
||||||
|
class TestRequiresStopSwarm(unittest.TestCase):
|
||||||
|
@patch("baudolo.backup.app.get_image_info")
|
||||||
|
@patch("baudolo.backup.app.is_swarm_task", return_value=True)
|
||||||
|
def test_swarm_tasks_never_require_stop(
|
||||||
|
self, _mock_is_swarm_task, mock_get_image_info
|
||||||
|
):
|
||||||
|
mock_get_image_info.return_value = "repo/not-whitelisted:latest"
|
||||||
|
self.assertFalse(requires_stop(["c1"], []))
|
||||||
|
mock_get_image_info.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user