test(e2e): stabilize MariaDB 11 tests and fix restore empty-mode client selection

- add `make clean` and run it before `test-e2e` to avoid stale artifacts
- restore: do not hardcode `mysql` for --empty; use detected mariadb/mysql client
- e2e: improve subprocess error output for easier CI debugging
- e2e: adjust MariaDB readiness checks for socket-only root on MariaDB 11
- e2e: add `wait_for_mariadb_sql` and run SQL readiness checks for dedicated TCP user
- e2e: update MariaDB full/no-copy tests to use dedicated user over TCP and consistent credentials

https://chatgpt.com/share/694ecfb8-f8a8-800f-a1c9-a5f410d4ba02

tests/e2e/helpers.py

@@ -8,14 +8,30 @@ import uuid
 from pathlib import Path
 
 
-def run(cmd: list[str], *, capture: bool = True, check: bool = True, cwd: str | None = None) -> subprocess.CompletedProcess:
-    return subprocess.run(
-        cmd,
-        check=check,
-        cwd=cwd,
-        text=True,
-        capture_output=capture,
-    )
+def run(
+    cmd: list[str],
+    *,
+    capture: bool = True,
+    check: bool = True,
+    cwd: str | None = None,
+) -> subprocess.CompletedProcess:
+    try:
+        return subprocess.run(
+            cmd,
+            check=check,
+            cwd=cwd,
+            text=True,
+            capture_output=capture,
+        )
+    except subprocess.CalledProcessError as e:
+        # Print captured output so failing E2E tests are "live" / debuggable in CI logs
+        print(">>> command failed:", " ".join(cmd))
+        print(">>> exit code:", e.returncode)
+        if e.stdout:
+            print(">>> STDOUT:\n" + e.stdout)
+        if e.stderr:
+            print(">>> STDERR:\n" + e.stderr)
+        raise
 
 
 def sh(cmd: str, *, capture: bool = True, check: bool = True) -> subprocess.CompletedProcess:
@@ -66,13 +82,17 @@ def wait_for_postgres(container: str, *, user: str = "postgres", timeout_s: int
 
 def wait_for_mariadb(container: str, *, root_password: str, timeout_s: int = 90) -> None:
     """
-    Docker-outside-of-Docker friendly readiness: check from inside the DB container.
+    Liveness probe for MariaDB.
+
+    IMPORTANT (MariaDB 11):
+    Root TCP auth is often restricted (unix_socket auth), so a TCP ping like
+    `mariadb-admin -uroot -p... -h localhost ping` can fail even though the server is up.
+    We therefore check readiness via a socket-based query.
     """
     deadline = time.time() + timeout_s
     while time.time() < deadline:
-        # mariadb-admin is present in the official mariadb image
         p = run(
-            ["docker", "exec", container, "sh", "-lc", f"mariadb-admin -uroot -p{root_password} ping -h localhost"],
+            ["docker", "exec", container, "sh", "-lc", "mariadb -uroot --protocol=socket -e \"SELECT 1;\""],
             capture=True,
             check=False,
         )
@@ -82,6 +102,33 @@ def wait_for_mariadb(container: str, *, root_password: str, timeout_s: int = 90)
     raise TimeoutError(f"Timed out waiting for MariaDB readiness in container {container}")
 
 
+def wait_for_mariadb_sql(container: str, *, user: str, password: str, timeout_s: int = 90) -> None:
+    """
+    SQL login readiness for the *dedicated test user* over TCP.
+
+    This is separate from wait_for_mariadb(root) because root may be socket-only,
+    while the tests use a normal user that should work via TCP.
+    """
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        p = run(
+            [
+                "docker",
+                "exec",
+                container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{user} -p{password} -e \"SELECT 1;\"",
+            ],
+            capture=True,
+            check=False,
+        )
+        if p.returncode == 0:
+            return
+        time.sleep(1)
+    raise TimeoutError(f"Timed out waiting for MariaDB SQL login readiness in container {container}")
+
+
 def backup_run(
     *,
     backups_dir: str,
@@ -111,7 +158,6 @@ def backup_run(
     try:
         run(cmd, capture=True, check=True)
     except subprocess.CalledProcessError as e:
-        # Print captured output so failing E2E tests are "live" / debuggable in CI logs
         print(">>> baudolo failed (exit code:", e.returncode, ")")
         if e.stdout:
             print(">>> baudolo STDOUT:\n" + e.stdout)

tests/e2e/test_e2e_mariadb_full.py

@@ -13,6 +13,7 @@ from .helpers import (
     write_databases_csv,
     run,
     wait_for_mariadb,
+    wait_for_mariadb_sql,
 )
 
 
@@ -31,30 +32,56 @@ class TestE2EMariaDBFull(unittest.TestCase):
         cls.containers = [cls.db_container]
         cls.volumes = [cls.db_volume]
 
+        cls.db_name = "appdb"
+        cls.db_user = "test"
+        cls.db_password = "testpw"
+        cls.root_password = "rootpw"
+
         run(["docker", "volume", "create", cls.db_volume])
 
-        # Start MariaDB (no host port publishing needed; we will exec into the container)
-        run([
-            "docker", "run", "-d",
-            "--name", cls.db_container,
-            "-e", "MARIADB_ROOT_PASSWORD=rootpw",
-            "-v", f"{cls.db_volume}:/var/lib/mysql",
-            "mariadb:11",
-        ])
-        wait_for_mariadb(cls.db_container, root_password="rootpw", timeout_s=90)
+        # Start MariaDB with a dedicated TCP-capable user for tests.
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.db_container,
+                "-e",
+                f"MARIADB_ROOT_PASSWORD={cls.root_password}",
+                "-e",
+                f"MARIADB_DATABASE={cls.db_name}",
+                "-e",
+                f"MARIADB_USER={cls.db_user}",
+                "-e",
+                f"MARIADB_PASSWORD={cls.db_password}",
+                "-v",
+                f"{cls.db_volume}:/var/lib/mysql",
+                "mariadb:11",
+            ]
+        )
 
-        # Create DB + data
-        run([
-            "docker", "exec", cls.db_container,
-            "sh", "-lc",
-            "mariadb -uroot -prootpw -e \"CREATE DATABASE appdb; "
-            "CREATE TABLE appdb.t (id INT PRIMARY KEY, v VARCHAR(50)); "
-            "INSERT INTO appdb.t VALUES (1,'ok');\"",
-        ])
+        # Liveness + actual SQL login readiness (TCP)
+        wait_for_mariadb(cls.db_container, root_password=cls.root_password, timeout_s=90)
+        wait_for_mariadb_sql(cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90)
+
+        # Create table + data via the dedicated user (TCP)
+        run(
+            [
+                "docker",
+                "exec",
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f"-e \"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); "
+                f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
+            ]
+        )
 
         cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
-        instance = cls.db_container
-        write_databases_csv(cls.databases_csv, [(instance, "appdb", "root", "rootpw")])
+        # IMPORTANT: baudolo backup expects credentials for the DB dump.
+        write_databases_csv(cls.databases_csv, [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)])
 
         # Backup with file+dump
         backup_run(
@@ -68,38 +95,61 @@ class TestE2EMariaDBFull(unittest.TestCase):
 
         cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
 
-        # Wipe DB
-        run([
-            "docker", "exec", cls.db_container,
-            "sh", "-lc",
-            "mariadb -uroot -prootpw -e \"DROP DATABASE appdb;\"",
-        ])
+        # Wipe DB via the dedicated user (TCP)
+        run(
+            [
+                "docker",
+                "exec",
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f"-e \"DROP TABLE {cls.db_name}.t;\"",
+            ]
+        )
 
-        # Restore DB
-        run([
-            "baudolo-restore", "mariadb",
-            cls.db_volume, cls.hash, cls.version,
-            "--backups-dir", cls.backups_dir,
-            "--repo-name", cls.repo_name,
-            "--container", cls.db_container,
-            "--db-name", "appdb",
-            "--db-user", "root",
-            "--db-password", "rootpw",
-            "--empty",
-        ])
+        # Restore DB (uses baudolo-restore which execs mysql/mariadb inside the container)
+        run(
+            [
+                "baudolo-restore",
+                "mariadb",
+                cls.db_volume,
+                cls.hash,
+                cls.version,
+                "--backups-dir",
+                cls.backups_dir,
+                "--repo-name",
+                cls.repo_name,
+                "--container",
+                cls.db_container,
+                "--db-name",
+                cls.db_name,
+                "--db-user",
+                cls.db_user,
+                "--db-password",
+                cls.db_password,
+                "--empty",
+            ]
+        )
 
     @classmethod
     def tearDownClass(cls) -> None:
         cleanup_docker(containers=cls.containers, volumes=cls.volumes)
 
     def test_dump_file_exists(self) -> None:
-        p = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume) / "sql" / "appdb.backup.sql"
+        p = backup_path(self.backups_dir, self.repo_name, self.version, self.db_volume) / "sql" / f"{self.db_name}.backup.sql"
         self.assertTrue(p.is_file(), f"Expected dump file at: {p}")
 
     def test_data_restored(self) -> None:
-        p = run([
-            "docker", "exec", self.db_container,
-            "sh", "-lc",
-            "mariadb -uroot -prootpw -N -e \"SELECT v FROM appdb.t WHERE id=1;\"",
-        ])
+        p = run(
+            [
+                "docker",
+                "exec",
+                self.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
+                f"-N -e \"SELECT v FROM {self.db_name}.t WHERE id=1;\"",
+            ]
+        )
         self.assertEqual((p.stdout or "").strip(), "ok")

tests/e2e/test_e2e_mariadb_no_copy.py

@@ -13,6 +13,7 @@ from .helpers import (
     write_databases_csv,
     run,
     wait_for_mariadb,
+    wait_for_mariadb_sql,
 )
 
 
@@ -31,26 +32,53 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
         cls.containers = [cls.db_container]
         cls.volumes = [cls.db_volume]
 
-        run(["docker", "volume", "create", cls.db_volume])
-        run([
-            "docker", "run", "-d",
-            "--name", cls.db_container,
-            "-e", "MARIADB_ROOT_PASSWORD=rootpw",
-            "-v", f"{cls.db_volume}:/var/lib/mysql",
-            "mariadb:11",
-        ])
-        wait_for_mariadb(cls.db_container, root_password="rootpw", timeout_s=90)
+        cls.db_name = "appdb"
+        cls.db_user = "test"
+        cls.db_password = "testpw"
+        cls.root_password = "rootpw"
 
-        run([
-            "docker", "exec", cls.db_container,
-            "sh", "-lc",
-            "mariadb -uroot -prootpw -e \"CREATE DATABASE appdb; "
-            "CREATE TABLE appdb.t (id INT PRIMARY KEY, v VARCHAR(50)); "
-            "INSERT INTO appdb.t VALUES (1,'ok');\"",
-        ])
+        run(["docker", "volume", "create", cls.db_volume])
+
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.db_container,
+                "-e",
+                f"MARIADB_ROOT_PASSWORD={cls.root_password}",
+                "-e",
+                f"MARIADB_DATABASE={cls.db_name}",
+                "-e",
+                f"MARIADB_USER={cls.db_user}",
+                "-e",
+                f"MARIADB_PASSWORD={cls.db_password}",
+                "-v",
+                f"{cls.db_volume}:/var/lib/mysql",
+                "mariadb:11",
+            ]
+        )
+
+        wait_for_mariadb(cls.db_container, root_password=cls.root_password, timeout_s=90)
+        wait_for_mariadb_sql(cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=90)
+
+        # Create table + data (TCP)
+        run(
+            [
+                "docker",
+                "exec",
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f"-e \"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50)); "
+                f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');\"",
+            ]
+        )
 
         cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
-        write_databases_csv(cls.databases_csv, [(cls.db_container, "appdb", "root", "rootpw")])
+        write_databases_csv(cls.databases_csv, [(cls.db_container, cls.db_name, cls.db_user, cls.db_password)])
 
         # dump-only => no files
         backup_run(
@@ -65,25 +93,42 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
 
         cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
 
-        # Wipe DB
-        run([
-            "docker", "exec", cls.db_container,
-            "sh", "-lc",
-            "mariadb -uroot -prootpw -e \"DROP DATABASE appdb;\"",
-        ])
+        # Wipe table (TCP)
+        run(
+            [
+                "docker",
+                "exec",
+                cls.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{cls.db_user} -p{cls.db_password} "
+                f"-e \"DROP TABLE {cls.db_name}.t;\"",
+            ]
+        )
 
         # Restore DB
-        run([
-            "baudolo-restore", "mariadb",
-            cls.db_volume, cls.hash, cls.version,
-            "--backups-dir", cls.backups_dir,
-            "--repo-name", cls.repo_name,
-            "--container", cls.db_container,
-            "--db-name", "appdb",
-            "--db-user", "root",
-            "--db-password", "rootpw",
-            "--empty",
-        ])
+        run(
+            [
+                "baudolo-restore",
+                "mariadb",
+                cls.db_volume,
+                cls.hash,
+                cls.version,
+                "--backups-dir",
+                cls.backups_dir,
+                "--repo-name",
+                cls.repo_name,
+                "--container",
+                cls.db_container,
+                "--db-name",
+                cls.db_name,
+                "--db-user",
+                cls.db_user,
+                "--db-password",
+                cls.db_password,
+                "--empty",
+            ]
+        )
 
     @classmethod
     def tearDownClass(cls) -> None:
@@ -94,9 +139,15 @@ class TestE2EMariaDBNoCopy(unittest.TestCase):
         self.assertFalse(p.exists(), f"Did not expect files backup dir at: {p}")
 
     def test_data_restored(self) -> None:
-        p = run([
-            "docker", "exec", self.db_container,
-            "sh", "-lc",
-            "mariadb -uroot -prootpw -N -e \"SELECT v FROM appdb.t WHERE id=1;\"",
-        ])
+        p = run(
+            [
+                "docker",
+                "exec",
+                self.db_container,
+                "sh",
+                "-lc",
+                f"mariadb -h 127.0.0.1 -u{self.db_user} -p{self.db_password} "
+                f"-N -e \"SELECT v FROM {self.db_name}.t WHERE id=1;\"",
+            ]
+        )
         self.assertEqual((p.stdout or "").strip(), "ok")