test(e2e): stabilize MariaDB 11 tests and fix restore empty-mode client selection

- add `make clean` and run it before `test-e2e` to avoid stale artifacts
- restore: do not hardcode `mysql` for --empty; use detected mariadb/mysql client
- e2e: improve subprocess error output for easier CI debugging
- e2e: adjust MariaDB readiness checks for socket-only root on MariaDB 11
- e2e: add `wait_for_mariadb_sql` and run SQL readiness checks for dedicated TCP user
- e2e: update MariaDB full/no-copy tests to use dedicated user over TCP and consistent credentials

https://chatgpt.com/share/694ecfb8-f8a8-800f-a1c9-a5f410d4ba02

Makefile

@@ -29,6 +29,9 @@ build:
 	@echo ">> Building Docker image $(IMAGE)"
 	docker build -t $(IMAGE) .
 
+clean:
+	git clean -fdX .
+
 # ------------------------------------------------------------
 # Run E2E tests inside the container (Docker socket required)
 # ------------------------------------------------------------
@@ -37,5 +40,5 @@ build:
 # - starts a DinD daemon container on a dedicated network
 # - loads the freshly built image into DinD
 # - runs the unittest suite inside a container that talks to DinD via DOCKER_HOST
-test-e2e: build
+test-e2e: clean build
 	@bash scripts/test-e2e.sh

src/baudolo/restore/db/mariadb.py

@@ -7,7 +7,10 @@ from ..run import docker_exec, docker_exec_sh
 
 
 def _pick_client(container: str) -> str:
-    # Prefer mariadb, fallback to mysql (many images provide one or both).
+    """
+    Prefer 'mariadb', fallback to 'mysql'.
+    Some MariaDB images no longer ship a 'mysql' binary, so we must not assume it exists.
+    """
     script = r"""
 set -eu
 if command -v mariadb >/dev/null 2>&1; then echo mariadb; exit 0; fi
@@ -39,16 +42,18 @@ def restore_mariadb_sql(
         raise FileNotFoundError(sql_path)
 
     if empty:
-        # Close to old behavior: list tables, drop individually, then re-enable checks.
+        # IMPORTANT:
+        # Do NOT hardcode 'mysql' here. Use the detected client.
+        # MariaDB 11 images may not contain the mysql binary at all.
         docker_exec(
             container,
-            ["mysql", "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=0;"],
+            [client, "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=0;"],
         )
 
         result = docker_exec(
             container,
             [
-                "mysql",
+                client,
                 "-u",
                 user,
                 f"--password={password}",
@@ -59,11 +64,12 @@ def restore_mariadb_sql(
             capture=True,
         )
         tables = result.stdout.decode().split()
+
         for tbl in tables:
             docker_exec(
                 container,
                 [
-                    "mysql",
+                    client,
                     "-u",
                     user,
                     f"--password={password}",
@@ -74,7 +80,7 @@ def restore_mariadb_sql(
 
         docker_exec(
             container,
-            ["mysql", "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=1;"],
+            [client, "-u", user, f"--password={password}", "-e", "SET FOREIGN_KEY_CHECKS=1;"],
         )
 
     with open(sql_path, "rb") as f:

tests/e2e/helpers.py

@@ -8,14 +8,30 @@ import uuid
 from pathlib import Path
 
 
-def run(cmd: list[str], *, capture: bool = True, check: bool = True, cwd: str | None = None) -> subprocess.CompletedProcess:
+def run(
-    return subprocess.run(
+    cmd: list[str],
-        cmd,
+    *,
-        check=check,
+    capture: bool = True,
-        cwd=cwd,
+    check: bool = True,
-        text=True,
+    cwd: str | None = None,
-        capture_output=capture,
+) -> subprocess.CompletedProcess:
-    )
+    try:
+        return subprocess.run(
+            cmd,
+            check=check,
+            cwd=cwd,
+            text=True,
+            capture_output=capture,
+        )
+    except subprocess.CalledProcessError as e:
+        # Print captured output so failing E2E tests are "live" / debuggable in CI logs
+        print(">>> command failed:", " ".join(cmd))
+        print(">>> exit code:", e.returncode)
+        if e.stdout:
+            print(">>> STDOUT:\n" + e.stdout)
+        if e.stderr:
+            print(">>> STDERR:\n" + e.stderr)
+        raise
 
 
 def sh(cmd: str, *, capture: bool = True, check: bool = True) -> subprocess.CompletedProcess:
@@ -66,13 +82,17 @@ def wait_for_postgres(container: str, *, user: str = "postgres", timeout_s: int
 
 def wait_for_mariadb(container: str, *, root_password: str, timeout_s: int = 90) -> None:
     """
-    Docker-outside-of-Docker friendly readiness: check from inside the DB container.
+    Liveness probe for MariaDB.
+
+    IMPORTANT (MariaDB 11):
+    Root TCP auth is often restricted (unix_socket auth), so a TCP ping like
+    `mariadb-admin -uroot -p... -h localhost ping` can fail even though the server is up.
+    We therefore check readiness via a socket-based query.
     """
     deadline = time.time() + timeout_s
     while time.time() < deadline:
-        # mariadb-admin is present in the official mariadb image
         p = run(
-            ["docker", "exec", container, "sh", "-lc", f"mariadb-admin -uroot -p{root_password} ping -h localhost"],
+            ["docker", "exec", container, "sh", "-lc", "mariadb -uroot --protocol=socket -e \"SELECT 1;\""],
             capture=True,
             check=False,
         )
@@ -82,6 +102,33 @@ def wait_for_mariadb(container: str, *, root_password: str, timeout_s: int = 90)
     raise TimeoutError(f"Timed out waiting for MariaDB readiness in container {container}")
 
 
+def wait_for_mariadb_sql(container: str, *, user: str, password: str, timeout_s: int = 90) -> None:
+    """
+    SQL login readiness for the *dedicated test user* over TCP.
+
+    This is separate from wait_for_mariadb(root) because root may be socket-only,
+    while the tests use a normal user that should work via TCP.
+    """
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        p = run(
+            [
+                "docker",
+                "exec",
+                container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{user} -p{password} -e \"SELECT 1;\"",
+            ],
+            capture=True,
+            check=False,
+        )
+        if p.returncode == 0:
+            return
+        time.sleep(1)
+    raise TimeoutError(f"Timed out waiting for MariaDB SQL login readiness in container {container}")
+
+
 def backup_run(
     *,
     backups_dir: str,
@@ -111,7 +158,6 @@ def backup_run(
     try:
         run(cmd, capture=True, check=True)
     except subprocess.CalledProcessError as e:
-        # Print captured output so failing E2E tests are "live" / debuggable in CI logs
         print(">>> baudolo failed (exit code:", e.returncode, ")")
         if e.stdout:
             print(">>> baudolo STDOUT:\n" + e.stdout)

tests/e2e/test_e2e_mariadb_full.py

@@ -13,6 +13,7 @@ from .helpers import (
     write_databases_csv,
     run,
     wait_for_mariadb,
+    wait_for_mariadb_sql,
 )
 
 
@@ -31,30 +32,56 @@ class TestE2EMariaDBFull(unittest.TestCase):
         cls.containers = [cls.db_container]
         cls.volumes = [cls.db_volume]
 
+        cls.db_name = "appdb"
+        cls.db_user = "test"
+        cls.db_password = "testpw"
+        cls.root_password = "rootpw"
+
         run(["docker", "volume", "create", cls.db_volume])
 
-        # Start MariaDB (no host port publishing needed; we will exec into the container)
+        # Start MariaDB with a dedicated TCP-capable user for tests.
-        run([
+        run(
-            "docker", "run", "-d",
+            [
-            "--name", cls.db_container,
+                "docker",
-            "-e", "MARIADB_ROOT_PASSWORD=rootpw",
+                "run",
-            "-v", f"{cls.db_volume}:/var/lib/mysql",
+                "-d",
-            "mariadb:11",
+                "--name",
-        ])
+                cls.db_container,
-        wait_for_mariadb(cls.db_container, root_password="rootpw", timeout_s=90)
+                "-e",
+                f"MARIADB_ROOT_PASSWORD={cls.root_password}",
+                "-e",
+                f"MARIADB_DATABASE={cls.db_name}",
+                "-e",
+                f"MARIADB_USER={cls.db_user}",
+                "-e",
+                f"MARIADB_PASSWORD={cls.db_password}",
+                "-v",
+                f"{cls.db_volume}:/var/lib/mysql",
+                "mariadb:11",
+            ]
+        )
 
-        # Create DB + data
+        # Liveness + actual SQL login readiness (TCP)
-        run([
+        wait_for_mariadb(cls.db_container, root_password=cls.root_password, timeout_s=90)
-            "docker", "exec", cls.db_container,
+        wait_for_mariadb_sql(cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90)
-            "sh", "-lc",
+
-            "mariadb -uroot -prootpw -e \"CREATE DATABASE appdb; "
+        # Create table + data via the dedicated user (TCP)
-            "CREATE TABLE appdb.t (id INT PRIMARY KEY, v VARCHAR(50)); "
+        run(
-            "INSERT INTO appdb.t VALUES (1,'ok');\"",
+            [
-        ])
+                "docker",
+                "exec",
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f"-e \"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); "
+                f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
+            ]
+        )
 
         cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
-        instance = cls.db_container
+        # IMPORTANT: baudolo backup expects credentials for the DB dump.
-        write_databases_csv(cls.databases_csv, [(instance, "appdb", "root", "rootpw")])
+        write_databases_csv(cls.databases_csv, [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)])
 
         # Backup with file+dump
         backup_run(
@@ -68,38 +95,61 @@ class TestE2EMariaDBFull(unittest.TestCase):
 
         cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
 
-        # Wipe DB
+        # Wipe DB via the dedicated user (TCP)
-        run([
+        run(
-            "docker", "exec", cls.db_container,
+            [
-            "sh", "-lc",
+                "docker",
-            "mariadb -uroot -prootpw -e \"DROP DATABASE appdb;\"",
+                "exec",
-        ])
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f"-e \"DROP TABLE {cls.db_name}.t;\"",
+            ]
+        )
 
-        # Restore DB
+        # Restore DB (uses baudolo-restore which execs mysql/mariadb inside the container)
-        run([
+        run(
-            "baudolo-restore", "mariadb",
+            [
-            cls.db_volume, cls.hash, cls.version,
+                "baudolo-restore",
-            "--backups-dir", cls.backups_dir,
+                "mariadb",
-            "--repo-name", cls.repo_name,
+                cls.db_volume,
-            "--container", cls.db_container,
+                cls.hash,
-            "--db-name", "appdb",
+                cls.version,
-            "--db-user", "root",
+                "--backups-dir",
-            "--db-password", "rootpw",
+                cls.backups_dir,
-            "--empty",
+                "--repo-name",
-        ])
+                cls.repo_name,
+                "--container",
+                cls.db_container,
+                "--db-name",
+                cls.db_name,
+                "--db-user",
+                cls.db_user,
+                "--db-password",
+                cls.db_password,
+                "--empty",
+            ]
+        )
 
     @classmethod
     def tearDownClass(cls) -> None:
         cleanup_docker(containers=cls.containers, volumes=cls.volumes)
 
     def test_dump_file_exists(self) -> None:
-        p = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume) / "sql" / "appdb.backup.sql"
+        p = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume) / "sql" / f"{self.db_name}.backup.sql"
         self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
 
     def test_data_restored(self) -> None:
-        p = run([
+        p = run(
-            "docker", "exec", self.db_container,
+            [
-            "sh", "-lc",
+                "docker",
-            "mariadb -uroot -prootpw -N -e \"SELECT v FROM appdb.t WHERE id=1;\"",
+                "exec",
-        ])
+                self.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
+                f"-N -e \"SELECT v FROM {self.db_name}.t WHERE id=1;\"",
+            ]
+        )
         self.assertEqual((p.stdout or "").strip(), "ok")

tests/e2e/test_e2e_mariadb_no_copy.py

@@ -13,6 +13,7 @@ from .helpers import (
     write_databases_csv,
     run,
     wait_for_mariadb,
+    wait_for_mariadb_sql,
 )
 
 
@@ -31,26 +32,53 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
         cls.containers = [cls.db_container]
         cls.volumes = [cls.db_volume]
 
-        run(["docker", "volume", "create", cls.db_volume])
+        cls.db_name = "appdb"
-        run([
+        cls.db_user = "test"
-            "docker", "run", "-d",
+        cls.db_password = "testpw"
-            "--name", cls.db_container,
+        cls.root_password = "rootpw"
-            "-e", "MARIADB_ROOT_PASSWORD=rootpw",
-            "-v", f"{cls.db_volume}:/var/lib/mysql",
-            "mariadb:11",
-        ])
-        wait_for_mariadb(cls.db_container, root_password="rootpw", timeout_s=90)
 
-        run([
+        run(["docker", "volume", "create", cls.db_volume])
-            "docker", "exec", cls.db_container,
+
-            "sh", "-lc",
+        run(
-            "mariadb -uroot -prootpw -e \"CREATE DATABASE appdb; "
+            [
-            "CREATE TABLE appdb.t (id INT PRIMARY KEY, v VARCHAR(50)); "
+                "docker",
-            "INSERT INTO appdb.t VALUES (1,'ok');\"",
+                "run",
-        ])
+                "-d",
+                "--name",
+                cls.db_container,
+                "-e",
+                f"MARIADB_ROOT_PASSWORD={cls.root_password}",
+                "-e",
+                f"MARIADB_DATABASE={cls.db_name}",
+                "-e",
+                f"MARIADB_USER={cls.db_user}",
+                "-e",
+                f"MARIADB_PASSWORD={cls.db_password}",
+                "-v",
+                f"{cls.db_volume}:/var/lib/mysql",
+                "mariadb:11",
+            ]
+        )
+
+        wait_for_mariadb(cls.db_container, root_password=cls.root_password, timeout_s=90)
+        wait_for_mariadb_sql(cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90)
+
+        # Create table + data (TCP)
+        run(
+            [
+                "docker",
+                "exec",
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f"-e \"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); "
+                f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
+            ]
+        )
 
         cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
-        write_databases_csv(cls.databases_csv, [(cls.db_container, "appdb", "root", "rootpw")])
+        write_databases_csv(cls.databases_csv, [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)])
 
         # dump-only => no files
         backup_run(
@@ -65,25 +93,42 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
 
         cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
 
-        # Wipe DB
+        # Wipe table (TCP)
-        run([
+        run(
-            "docker", "exec", cls.db_container,
+            [
-            "sh", "-lc",
+                "docker",
-            "mariadb -uroot -prootpw -e \"DROP DATABASE appdb;\"",
+                "exec",
-        ])
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f"-e \"DROP TABLE {cls.db_name}.t;\"",
+            ]
+        )
 
         # Restore DB
-        run([
+        run(
-            "baudolo-restore", "mariadb",
+            [
-            cls.db_volume, cls.hash, cls.version,
+                "baudolo-restore",
-            "--backups-dir", cls.backups_dir,
+                "mariadb",
-            "--repo-name", cls.repo_name,
+                cls.db_volume,
-            "--container", cls.db_container,
+                cls.hash,
-            "--db-name", "appdb",
+                cls.version,
-            "--db-user", "root",
+                "--backups-dir",
-            "--db-password", "rootpw",
+                cls.backups_dir,
-            "--empty",
+                "--repo-name",
-        ])
+                cls.repo_name,
+                "--container",
+                cls.db_container,
+                "--db-name",
+                cls.db_name,
+                "--db-user",
+                cls.db_user,
+                "--db-password",
+                cls.db_password,
+                "--empty",
+            ]
+        )
 
     @classmethod
     def tearDownClass(cls) -> None:
@@ -94,9 +139,15 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
         self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
 
     def test_data_restored(self) -> None:
-        p = run([
+        p = run(
-            "docker", "exec", self.db_container,
+            [
-            "sh", "-lc",
+                "docker",
-            "mariadb -uroot -prootpw -N -e \"SELECT v FROM appdb.t WHERE id=1;\"",
+                "exec",
-        ])
+                self.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
+                f"-N -e \"SELECT v FROM {self.db_name}.t WHERE id=1;\"",
+            ]
+        )
         self.assertEqual((p.stdout or "").strip(), "ok")