diff --git a/Makefile b/Makefile index 36b9234..f16b0ba 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,9 @@ build: @echo ">> Building Docker image $(IMAGE)" docker build -t $(IMAGE) . +clean: + git clean -fdX . + # ------------------------------------------------------------ # Run E2E tests inside the container (Docker socket required) # ------------------------------------------------------------ @@ -37,5 +40,5 @@ build: # - starts a DinD daemon container on a dedicated network # - loads the freshly built image into DinD # - runs the unittest suite inside a container that talks to DinD via DOCKER_HOST -test-e2e: build +test-e2e: clean build @bash scripts/test-e2e.sh \ No newline at end of file diff --git a/src/baudolo/restore/db/mariadb.py b/src/baudolo/restore/db/mariadb.py index ab3daa6..17f7e04 100644 --- a/src/baudolo/restore/db/mariadb.py +++ b/src/baudolo/restore/db/mariadb.py @@ -7,7 +7,10 @@ from ..run import docker_exec, docker_exec_sh def _pick_client(container: str) -> str: - # Prefer mariadb, fallback to mysql (many images provide one or both). + """ + Prefer 'mariadb', fallback to 'mysql'. + Some MariaDB images no longer ship a 'mysql' binary, so we must not assume it exists. + """ script = r""" set -eu if command -v mariadb >/dev/null 2>&1; then echo mariadb; exit 0; fi @@ -39,16 +42,18 @@ def restore_mariadb_sql( raise FileNotFoundError(sql_path) if empty: - # Close to old behavior: list tables, drop individually, then re-enable checks. + # IMPORTANT: + # Do NOT hardcode 'mysql' here. Use the detected client. + # MariaDB 11 images may not contain the mysql binary at all. docker_exec( container, - ["mysql", "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=0;"], + [client, "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=0;"], ) result = docker_exec( container, [ - "mysql", + client, "-u", user, f"--password={password}", @@ -59,11 +64,12 @@ def restore_mariadb_sql( capture=True, ) tables = result.stdout.decode().split() + for tbl in tables: docker_exec( container, [ - "mysql", + client, "-u", user, f"--password={password}", @@ -74,7 +80,7 @@ def restore_mariadb_sql( docker_exec( container, - ["mysql", "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=1;"], + [client, "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=1;"], ) with open(sql_path, "rb") as f: diff --git a/tests/e2e/helpers.py b/tests/e2e/helpers.py index 2412e60..16ab520 100644 --- a/tests/e2e/helpers.py +++ b/tests/e2e/helpers.py @@ -8,14 +8,30 @@ import uuid from pathlib import Path -def run(cmd: list[str], *, capture: bool = True, check: bool = True, cwd: str | None = None) -> subprocess.CompletedProcess: - return subprocess.run( - cmd, - check=check, - cwd=cwd, - text=True, - capture_output=capture, - ) +def run( + cmd: list[str], + *, + capture: bool = True, + check: bool = True, + cwd: str | None = None, +) -> subprocess.CompletedProcess: + try: + return subprocess.run( + cmd, + check=check, + cwd=cwd, + text=True, + capture_output=capture, + ) + except subprocess.CalledProcessError as e: + # Print captured output so failing E2E tests are "live" / debuggable in CI logs + print(">>> command failed:", " ".join(cmd)) + print(">>> exit code:", e.returncode) + if e.stdout: + print(">>> STDOUT:\n" + e.stdout) + if e.stderr: + print(">>> STDERR:\n" + e.stderr) + raise def sh(cmd: str, *, capture: bool = True, check: bool = True) -> subprocess.CompletedProcess: @@ -66,13 +82,17 @@ def wait_for_postgres(container: str, *, user: str = "postgres", timeout_s: int def wait_for_mariadb(container: str, *, root_password: str, timeout_s: int = 90) -> None: """ - Docker-outside-of-Docker friendly readiness: check from inside the DB container. + Liveness probe for MariaDB. + + IMPORTANT (MariaDB 11): + Root TCP auth is often restricted (unix_socket auth), so a TCP ping like + `mariadb-admin -uroot -p... -h localhost ping` can fail even though the server is up. + We therefore check readiness via a socket-based query. """ deadline = time.time() + timeout_s while time.time() < deadline: - # mariadb-admin is present in the official mariadb image p = run( - ["docker", "exec", container, "sh", "-lc", f"mariadb-admin -uroot -p{root_password} ping -h localhost"], + ["docker", "exec", container, "sh", "-lc", "mariadb -uroot --protocol=socket -e \"SELECT 1;\""], capture=True, check=False, ) @@ -82,6 +102,33 @@ def wait_for_mariadb(container: str, *, root_password: str, timeout_s: int = 90) raise TimeoutError(f"Timed out waiting for MariaDB readiness in container {container}") +def wait_for_mariadb_sql(container: str, *, user: str, password: str, timeout_s: int = 90) -> None: + """ + SQL login readiness for the *dedicated test user* over TCP. + + This is separate from wait_for_mariadb(root) because root may be socket-only, + while the tests use a normal user that should work via TCP. + """ + deadline = time.time() + timeout_s + while time.time() < deadline: + p = run( + [ + "docker", + "exec", + container, + "sh", + "-lc", + f"mariadb -h 127.0.0.1 -u{user} -p{password} -e \"SELECT 1;\"", + ], + capture=True, + check=False, + ) + if p.returncode == 0: + return + time.sleep(1) + raise TimeoutError(f"Timed out waiting for MariaDB SQL login readiness in container {container}") + + def backup_run( *, backups_dir: str, @@ -111,7 +158,6 @@ def backup_run( try: run(cmd, capture=True, check=True) except subprocess.CalledProcessError as e: - # Print captured output so failing E2E tests are "live" / debuggable in CI logs print(">>> baudolo failed (exit code:", e.returncode, ")") if e.stdout: print(">>> baudolo STDOUT:\n" + e.stdout) diff --git a/tests/e2e/test_e2e_mariadb_full.py b/tests/e2e/test_e2e_mariadb_full.py index 620a145..108ce2e 100644 --- a/tests/e2e/test_e2e_mariadb_full.py +++ b/tests/e2e/test_e2e_mariadb_full.py @@ -13,6 +13,7 @@ from .helpers import ( write_databases_csv, run, wait_for_mariadb, + wait_for_mariadb_sql, ) @@ -31,30 +32,56 @@ class TestE2EMariaDBFull(unittest.TestCase): cls.containers = [cls.db_container] cls.volumes = [cls.db_volume] + cls.db_name = "appdb" + cls.db_user = "test" + cls.db_password = "testpw" + cls.root_password = "rootpw" + run(["docker", "volume", "create", cls.db_volume]) - # Start MariaDB (no host port publishing needed; we will exec into the container) - run([ - "docker", "run", "-d", - "--name", cls.db_container, - "-e", "MARIADB_ROOT_PASSWORD=rootpw", - "-v", f"{cls.db_volume}:/var/lib/mysql", - "mariadb:11", - ]) - wait_for_mariadb(cls.db_container, root_password="rootpw", timeout_s=90) + # Start MariaDB with a dedicated TCP-capable user for tests. + run( + [ + "docker", + "run", + "-d", + "--name", + cls.db_container, + "-e", + f"MARIADB_ROOT_PASSWORD={cls.root_password}", + "-e", + f"MARIADB_DATABASE={cls.db_name}", + "-e", + f"MARIADB_USER={cls.db_user}", + "-e", + f"MARIADB_PASSWORD={cls.db_password}", + "-v", + f"{cls.db_volume}:/var/lib/mysql", + "mariadb:11", + ] + ) - # Create DB + data - run([ - "docker", "exec", cls.db_container, - "sh", "-lc", - "mariadb -uroot -prootpw -e \"CREATE DATABASE appdb; " - "CREATE TABLE appdb.t (id INT PRIMARY KEY, v VARCHAR(50)); " - "INSERT INTO appdb.t VALUES (1,'ok');\"", - ]) + # Liveness + actual SQL login readiness (TCP) + wait_for_mariadb(cls.db_container, root_password=cls.root_password, timeout_s=90) + wait_for_mariadb_sql(cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90) + + # Create table + data via the dedicated user (TCP) + run( + [ + "docker", + "exec", + cls.db_container, + "sh", + "-lc", + f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} " + f"-e \"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); " + f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"", + ] + ) cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv" - instance = cls.db_container - write_databases_csv(cls.databases_csv, [(instance, "appdb", "root", "rootpw")]) + # IMPORTANT: baudolo backup expects credentials for the DB dump. + write_databases_csv(cls.databases_csv, [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)]) # Backup with file+dump backup_run( @@ -68,38 +95,61 @@ class TestE2EMariaDBFull(unittest.TestCase): cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name) - # Wipe DB - run([ - "docker", "exec", cls.db_container, - "sh", "-lc", - "mariadb -uroot -prootpw -e \"DROP DATABASE appdb;\"", - ]) + # Wipe DB via the dedicated user (TCP) + run( + [ + "docker", + "exec", + cls.db_container, + "sh", + "-lc", + f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} " + f"-e \"DROP TABLE {cls.db_name}.t;\"", + ] + ) - # Restore DB - run([ - "baudolo-restore", "mariadb", - cls.db_volume, cls.hash, cls.version, - "--backups-dir", cls.backups_dir, - "--repo-name", cls.repo_name, - "--container", cls.db_container, - "--db-name", "appdb", - "--db-user", "root", - "--db-password", "rootpw", - "--empty", - ]) + # Restore DB (uses baudolo-restore which execs mysql/mariadb inside the container) + run( + [ + "baudolo-restore", + "mariadb", + cls.db_volume, + cls.hash, + cls.version, + "--backups-dir", + cls.backups_dir, + "--repo-name", + cls.repo_name, + "--container", + cls.db_container, + "--db-name", + cls.db_name, + "--db-user", + cls.db_user, + "--db-password", + cls.db_password, + "--empty", + ] + ) @classmethod def tearDownClass(cls) -> None: cleanup_docker(containers=cls.containers, volumes=cls.volumes) def test_dump_file_exists(self) -> None: - p = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume) / "sql" / "appdb.backup.sql" + p = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume) / "sql" / f"{self.db_name}.backup.sql" self.assertTrue(p.is_file(), f"Expected dump file at: {p}") def test_data_restored(self) -> None: - p = run([ - "docker", "exec", self.db_container, - "sh", "-lc", - "mariadb -uroot -prootpw -N -e \"SELECT v FROM appdb.t WHERE id=1;\"", - ]) + p = run( + [ + "docker", + "exec", + self.db_container, + "sh", + "-lc", + f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} " + f"-N -e \"SELECT v FROM {self.db_name}.t WHERE id=1;\"", + ] + ) self.assertEqual((p.stdout or "").strip(), "ok") diff --git a/tests/e2e/test_e2e_mariadb_no_copy.py b/tests/e2e/test_e2e_mariadb_no_copy.py index 47d362c..87736ab 100644 --- a/tests/e2e/test_e2e_mariadb_no_copy.py +++ b/tests/e2e/test_e2e_mariadb_no_copy.py @@ -13,6 +13,7 @@ from .helpers import ( write_databases_csv, run, wait_for_mariadb, + wait_for_mariadb_sql, ) @@ -31,26 +32,53 @@ class TestE2EMariaDBNoCopy(unittest.TestCase): cls.containers = [cls.db_container] cls.volumes = [cls.db_volume] - run(["docker", "volume", "create", cls.db_volume]) - run([ - "docker", "run", "-d", - "--name", cls.db_container, - "-e", "MARIADB_ROOT_PASSWORD=rootpw", - "-v", f"{cls.db_volume}:/var/lib/mysql", - "mariadb:11", - ]) - wait_for_mariadb(cls.db_container, root_password="rootpw", timeout_s=90) + cls.db_name = "appdb" + cls.db_user = "test" + cls.db_password = "testpw" + cls.root_password = "rootpw" - run([ - "docker", "exec", cls.db_container, - "sh", "-lc", - "mariadb -uroot -prootpw -e \"CREATE DATABASE appdb; " - "CREATE TABLE appdb.t (id INT PRIMARY KEY, v VARCHAR(50)); " - "INSERT INTO appdb.t VALUES (1,'ok');\"", - ]) + run(["docker", "volume", "create", cls.db_volume]) + + run( + [ + "docker", + "run", + "-d", + "--name", + cls.db_container, + "-e", + f"MARIADB_ROOT_PASSWORD={cls.root_password}", + "-e", + f"MARIADB_DATABASE={cls.db_name}", + "-e", + f"MARIADB_USER={cls.db_user}", + "-e", + f"MARIADB_PASSWORD={cls.db_password}", + "-v", + f"{cls.db_volume}:/var/lib/mysql", + "mariadb:11", + ] + ) + + wait_for_mariadb(cls.db_container, root_password=cls.root_password, timeout_s=90) + wait_for_mariadb_sql(cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90) + + # Create table + data (TCP) + run( + [ + "docker", + "exec", + cls.db_container, + "sh", + "-lc", + f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} " + f"-e \"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); " + f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"", + ] + ) cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv" - write_databases_csv(cls.databases_csv, [(cls.db_container, "appdb", "root", "rootpw")]) + write_databases_csv(cls.databases_csv, [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)]) # dump-only => no files backup_run( @@ -65,25 +93,42 @@ class TestE2EMariaDBNoCopy(unittest.TestCase): cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name) - # Wipe DB - run([ - "docker", "exec", cls.db_container, - "sh", "-lc", - "mariadb -uroot -prootpw -e \"DROP DATABASE appdb;\"", - ]) + # Wipe table (TCP) + run( + [ + "docker", + "exec", + cls.db_container, + "sh", + "-lc", + f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} " + f"-e \"DROP TABLE {cls.db_name}.t;\"", + ] + ) # Restore DB - run([ - "baudolo-restore", "mariadb", - cls.db_volume, cls.hash, cls.version, - "--backups-dir", cls.backups_dir, - "--repo-name", cls.repo_name, - "--container", cls.db_container, - "--db-name", "appdb", - "--db-user", "root", - "--db-password", "rootpw", - "--empty", - ]) + run( + [ + "baudolo-restore", + "mariadb", + cls.db_volume, + cls.hash, + cls.version, + "--backups-dir", + cls.backups_dir, + "--repo-name", + cls.repo_name, + "--container", + cls.db_container, + "--db-name", + cls.db_name, + "--db-user", + cls.db_user, + "--db-password", + cls.db_password, + "--empty", + ] + ) @classmethod def tearDownClass(cls) -> None: @@ -94,9 +139,15 @@ class TestE2EMariaDBNoCopy(unittest.TestCase): self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}") def test_data_restored(self) -> None: - p = run([ - "docker", "exec", self.db_container, - "sh", "-lc", - "mariadb -uroot -prootpw -N -e \"SELECT v FROM appdb.t WHERE id=1;\"", - ]) + p = run( + [ + "docker", + "exec", + self.db_container, + "sh", + "-lc", + f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} " + f"-N -e \"SELECT v FROM {self.db_name}.t WHERE id=1;\"", + ] + ) self.assertEqual((p.stdout or "").strip(), "ok")