5 Commits

Author SHA1 Message Date
57ea4592c1 Release version 1.7.1 2026-05-26 00:52:37 +02:00
ad5d8fcda3 fix(backup): force TCP for mariadb-dump to match '<user>'@'%' grant
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-26 00:46:47 +02:00
bfa596ae30 fix(test-e2e): set DinD MTU to 1280 to fix ghcr.io pull timeouts on broken-PMTUD host paths
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-26 00:45:29 +02:00
21b4d237d3 Release version 1.7.0 2026-02-07 14:00:11 +01:00
ec051b4c2b backup: support all valid docker compose file names
Detect compose files case-insensitively and support:
- compose.yml / compose.yaml
- docker-compose.yml / docker-compose.yaml

Replace hard-coded docker-compose.yml checks with a shared
finder helper and extend unit tests accordingly.

https://chatgpt.com/share/69873720-d444-800f-99f7-f7799fc10c0b
2026-02-07 13:58:52 +01:00
8 changed files with 310 additions and 11 deletions

View File

@@ -1,3 +1,15 @@
## [1.7.1] - 2026-05-26
* 🔌 MariaDB SQL backups now connect over TCP loopback so the dump always matches the same wildcard-host grant the application uses — no more surprise `ERROR 1045 Access denied` when a localhost-bound auth row preempts.
* 🧪 New regression and bug-repro tests pin the TCP behaviour and prove it under the exact preemption setup that caused the production failure on MariaDB 12.
* 🩺 E2E test infrastructure: DinD bridge and inner daemon now default to MTU 1280 so registry pulls survive host paths with broken PMTUD (override via `E2E_DIND_MTU`).
## [1.7.0] - 2026-02-07
* 🚀 Backup jobs now support all valid Docker Compose file names case-insensitive and hassle-free.
## [1.6.0] - 2026-02-06
* Compose handling is now fully delegated to the Infinito.Nexus compose wrapper or plain docker compose, removing all custom env and file detection to ensure a single, consistent source of truth.

View File

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "backup-docker-to-local"
version = "1.6.0"
version = "1.7.1"
description = "Backup Docker volumes to local with rsync and optional DB dumps."
readme = "README.md"
requires-python = ">=3.9"

View File

@@ -25,6 +25,8 @@ RSYNC_IMG="${E2E_RSYNC_IMAGE:-ghcr.io/kevinveenbirkenbach/alpine-rsync}"
READY_TIMEOUT_SECONDS="${E2E_READY_TIMEOUT_SECONDS:-120}"
ARTIFACTS_DIR="${E2E_ARTIFACTS_DIR:-./artifacts}"
DIND_MTU="${E2E_DIND_MTU:-1280}"
KEEP_ON_FAIL="${E2E_KEEP_ON_FAIL:-0}"
KEEP_VOLUMES="${E2E_KEEP_VOLUMES:-0}"
DEBUG_SHELL="${E2E_DEBUG_SHELL:-0}"
@@ -124,8 +126,11 @@ cleanup() {
}
trap cleanup EXIT INT TERM
log "Creating network ${NET} (if missing)"
docker network inspect "${NET}" >/dev/null 2>&1 || docker network create "${NET}" >/dev/null
log "(Re)creating network ${NET} with MTU ${DIND_MTU}"
docker network rm "${NET}" >/dev/null 2>&1 || true
docker network create \
--opt com.docker.network.driver.mtu="${DIND_MTU}" \
"${NET}" >/dev/null
log "Removing old ${DIND} (if any)"
docker rm -f "${DIND}" >/dev/null 2>&1 || true
@@ -148,7 +153,8 @@ docker run -d --privileged \
-p 2375:2375 \
docker:dind \
--host=tcp://0.0.0.0:2375 \
--tls=false >/dev/null
--tls=false \
--mtu="${DIND_MTU}" >/dev/null
log "Waiting for DinD to be ready..."
for i in $(seq 1 "${READY_TIMEOUT_SECONDS}"); do

View File

@@ -4,7 +4,7 @@ import os
import shutil
import subprocess
from pathlib import Path
from typing import List
from typing import List, Optional
def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
@@ -30,6 +30,37 @@ def _build_compose_cmd(project_dir: str, passthrough: List[str]) -> List[str]:
raise RuntimeError("Neither 'compose' nor 'docker' found in PATH")
def _find_compose_file(project_dir: str) -> Optional[Path]:
"""
Detect a compose file in `project_dir` (case-insensitive).
Supported names:
- compose.yml / compose.yaml
- docker-compose.yml / docker-compose.yaml
"""
pdir = Path(project_dir)
if not pdir.is_dir():
return None
# Map lowercase filename -> actual Path (preserves original casing)
by_lower = {p.name.lower(): p for p in pdir.iterdir() if p.is_file()}
# Preferred order (policy decision)
candidates = [
"docker-compose.yml",
"docker-compose.yaml",
"compose.yml",
"compose.yaml",
]
for name in candidates:
found = by_lower.get(name)
if found is not None:
return found
return None
def hard_restart_docker_services(dir_path: str) -> None:
print(f"Hard restart compose services in: {dir_path}", flush=True)
@@ -44,7 +75,8 @@ def hard_restart_docker_services(dir_path: str) -> None:
def handle_docker_compose_services(
parent_directory: str, hard_restart_required: list[str]
parent_directory: str,
hard_restart_required: list[str],
) -> None:
for entry in os.scandir(parent_directory):
if not entry.is_dir():
@@ -52,11 +84,12 @@ def handle_docker_compose_services(
dir_path = entry.path
name = os.path.basename(dir_path)
compose_file = os.path.join(dir_path, "docker-compose.yml")
print(f"Checking directory: {dir_path}", flush=True)
if not os.path.isfile(compose_file):
print("No docker-compose.yml found. Skipping.", flush=True)
compose_file = _find_compose_file(dir_path)
if compose_file is None:
print("No supported compose file found. Skipping.", flush=True)
continue
if name in hard_restart_required:

View File

@@ -115,8 +115,10 @@ def backup_database(
dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
if db_type == "mariadb":
# Force TCP so auth matches '<user>'@'%' instead of socket -> 'localhost'.
cmd = (
f"docker exec {container} /usr/bin/mariadb-dump "
f"-h 127.0.0.1 --protocol=tcp "
f"-u {user} -p{password} {db_name}"
)
_atomic_write_cmd(cmd, dump_file)

View File

@@ -0,0 +1,143 @@
"""
Bug-repro for: mariadb-dump fails with `ERROR 1045 Access denied for user
'<u>'@'localhost' (using password: YES)` when only '<u>'@'%' is granted and a
preempting ''@'localhost' user is present.
The fix forces TCP loopback in baudolo.backup.db so the dump matches the
'<u>'@'%' grant instead of the socket->localhost auth row.
This file:
- builds the exact preconditions that triggered the production failure,
- as a NEGATIVE control, runs a socket-based mariadb-dump (== the old code path)
and asserts that it fails with the literal 1045 / @'localhost' error,
- as a POSITIVE proof, calls backup_database() (where the fix lives) against
the same DB container and asserts the dump file is produced and contains the
seed data.
Note: the volume-rsync stage of baudolo is intentionally NOT exercised here.
That stage needs root on /var/lib/docker/volumes, which is provided by the
DinD wrapper in `make test-e2e` but not by an on-host invocation. The bug we
are verifying is in the DB-dump stage, so testing backup_database() directly
keeps the assertion focused and the test runnable both on-host and in DinD.
"""
import os
import tempfile
import unittest
import pandas
from baudolo.backup import db as db_mod
from .helpers import (
cleanup_docker,
require_docker,
run,
unique,
wait_for_mariadb,
wait_for_mariadb_sql,
)
class TestE2EMariaDBAnonymousPreemption(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
require_docker()
cls.prefix = unique("baudolo-e2e-mariadb-anon")
cls.db_container = f"{cls.prefix}-mariadb"
cls.db_volume = f"{cls.prefix}-mariadb-vol"
cls.containers = [cls.db_container]
cls.volumes = [cls.db_volume]
cls.db_name = "appdb"
cls.db_user = "tcponly"
cls.db_password = "tcponlypw"
cls.root_password = "rootpw"
run(["docker", "volume", "create", cls.db_volume])
# Boot WITHOUT MARIADB_USER/MARIADB_PASSWORD/MARIADB_DATABASE so the
# entrypoint does not auto-create '<u>'@'%'. We provision the user
# explicitly below to mirror the SQL path used by svc-db-mariadb.
run([
"docker", "run", "-d",
"--name", cls.db_container,
"-e", f"MARIADB_ROOT_PASSWORD={cls.root_password}",
"-v", f"{cls.db_volume}:/var/lib/mysql",
"mariadb:12.2",
])
wait_for_mariadb(cls.db_container, root_password=cls.root_password, timeout_s=120)
# Provision: '<u>'@'%' (the app/backup grant) + anonymous ''@'localhost'
# (the preemption trigger). Mirrors the production state that produced
# `ERROR 1045 ... '<u>'@'localhost' (using password: YES)`.
bootstrap_sql = (
f"CREATE DATABASE {cls.db_name};"
f"CREATE USER '{cls.db_user}'@'%' IDENTIFIED BY '{cls.db_password}';"
f"GRANT ALL PRIVILEGES ON {cls.db_name}.* TO '{cls.db_user}'@'%';"
f"CREATE USER ''@'localhost' IDENTIFIED BY 'anonpw-not-{cls.db_password}';"
"FLUSH PRIVILEGES;"
f"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50));"
f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');"
)
run([
"docker", "exec", cls.db_container, "sh", "-lc",
f'mariadb -uroot --protocol=socket -e "{bootstrap_sql}"',
])
# Sanity: '<u>' can log in over TCP (matches '%'). If THIS fails,
# the precondition for the fix to even apply is broken.
wait_for_mariadb_sql(
cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=60
)
@classmethod
def tearDownClass(cls) -> None:
cleanup_docker(containers=cls.containers, volumes=cls.volumes)
def test_negative_control_socket_dump_fails_with_1045(self) -> None:
# Reproduces the OLD code path (no -h/--protocol). MUST fail with 1045
# under the configured preemption. If this ever starts passing, either
# the MariaDB auth semantics changed or the anonymous-user setup did
# not take effect — in both cases the positive test below loses its
# ability to discriminate "fix works" vs "bug never reproduced".
p = run(
[
"docker", "exec", self.db_container, "sh", "-lc",
f"mariadb-dump -u{self.db_user} -p{self.db_password} {self.db_name}",
],
capture=True,
check=False,
)
self.assertNotEqual(p.returncode, 0, "socket-based dump unexpectedly succeeded")
self.assertIn("1045", (p.stderr or "") + (p.stdout or ""))
self.assertIn("@'localhost'", (p.stderr or "") + (p.stdout or ""))
def test_backup_database_succeeds_with_tcp_fix(self) -> None:
# Drives the function where the fix lives. No rsync, no privileged
# paths — just the dump that the negative-control proved is failing
# under the same preemption setup.
with tempfile.TemporaryDirectory() as volume_dir:
df = pandas.DataFrame(
[(self.db_container, self.db_name, self.db_user, self.db_password)],
columns=["instance", "database", "username", "password"],
)
produced = db_mod.backup_database(
container=self.db_container,
volume_dir=volume_dir,
db_type="mariadb",
databases_df=df,
database_containers=[self.db_container],
)
self.assertTrue(produced, "backup_database did not produce a dump")
dump_path = os.path.join(volume_dir, "sql", f"{self.db_name}.backup.sql")
self.assertTrue(os.path.isfile(dump_path), f"expected dump at {dump_path}")
with open(dump_path, "r", encoding="utf-8", errors="replace") as f:
content = f.read()
self.assertIn("INSERT INTO", content)
self.assertIn("'ok'", content)
if __name__ == "__main__":
unittest.main(verbosity=2)

View File

@@ -23,6 +23,7 @@ def _setup_compose_dir(
tmp_path: Path,
name: str = "mailu",
*,
compose_name: str = "docker-compose.yml",
with_override: bool = False,
with_ca_override: bool = False,
env_layout: str | None = None, # None | ".env" | ".env/env"
@@ -30,7 +31,7 @@ def _setup_compose_dir(
d = tmp_path / name
d.mkdir(parents=True, exist_ok=True)
_touch(d / "docker-compose.yml")
_touch(d / compose_name)
if with_override:
_touch(d / "docker-compose.override.yml")
@@ -53,11 +54,45 @@ class TestCompose(unittest.TestCase):
cls.compose_mod = mod
def test_find_compose_file_supports_all_valid_names_case_insensitive(self) -> None:
with tempfile.TemporaryDirectory() as td:
tmp_path = Path(td)
variants = [
"compose.yml",
"compose.yaml",
"docker-compose.yml",
"docker-compose.yaml",
"docker-compose.yAml",
]
for i, name in enumerate(variants):
d = _setup_compose_dir(
tmp_path,
name=f"project{i}",
compose_name=name,
)
found = self.compose_mod._find_compose_file(str(d))
self.assertIsNotNone(found)
self.assertEqual(found.name, name)
def test_find_compose_file_returns_none_when_missing(self) -> None:
with tempfile.TemporaryDirectory() as td:
tmp_path = Path(td)
d = tmp_path / "empty"
d.mkdir(parents=True, exist_ok=True)
found = self.compose_mod._find_compose_file(str(d))
self.assertIsNone(found)
def test_build_cmd_uses_wrapper_when_present(self) -> None:
with tempfile.TemporaryDirectory() as td:
tmp_path = Path(td)
d = _setup_compose_dir(
tmp_path, with_override=True, with_ca_override=True, env_layout=".env"
tmp_path,
with_override=True,
with_ca_override=True,
env_layout=".env",
)
def fake_which(name: str):

View File

@@ -0,0 +1,68 @@
import tempfile
import unittest
from unittest.mock import patch
import pandas
from baudolo.backup import db as db_mod
def _df(rows):
return pandas.DataFrame(
rows, columns=["instance", "database", "username", "password"]
)
def _capture_commands(*, db_type, rows, container):
captured = []
def _capture(cmd):
captured.append(cmd)
return []
with tempfile.TemporaryDirectory() as td:
with patch.object(db_mod, "execute_shell_command", side_effect=_capture):
db_mod.backup_database(
container=container,
volume_dir=td,
db_type=db_type,
databases_df=_df(rows),
database_containers=[container],
)
return captured
class TestMariaDBDumpUsesTCP(unittest.TestCase):
# Regression guard for 'Access denied for user <user>@localhost' when only
# '<user>'@'%' is granted: the in-container mariadb-dump MUST force TCP so
# the connection is auth-matched against '%' instead of socket->localhost.
def test_mariadb_dump_forces_tcp_loopback(self):
captured = _capture_commands(
db_type="mariadb",
rows=[("mariadb", "appdb", "appuser", "s3cret")],
container="mariadb",
)
dump_cmds = [c for c in captured if "mariadb-dump" in c]
self.assertEqual(len(dump_cmds), 1, f"expected one dump command, got: {captured}")
cmd = dump_cmds[0]
self.assertIn("-h 127.0.0.1", cmd)
self.assertIn("--protocol=tcp", cmd)
self.assertIn("-u appuser", cmd)
self.assertIn("-ps3cret", cmd)
self.assertIn(" appdb", cmd)
def test_postgres_dump_unaffected(self):
captured = _capture_commands(
db_type="postgres",
rows=[("pg", "appdb", "appuser", "s3cret")],
container="pg",
)
dump_cmds = [c for c in captured if "pg_dump" in c and "pg_dumpall" not in c]
self.assertEqual(len(dump_cmds), 1)
self.assertNotIn("--protocol=tcp", dump_cmds[0])
if __name__ == "__main__":
unittest.main(verbosity=2)