Release version 3.4.1

infinito-nexus-core gates `git commit` behind an explicit per-invocation
confirmation. This repository did not, so an agent working here could commit
without being asked. Mirror the rule.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.claude/settings.json

@@ -1,6 +1,7 @@
 {
   "permissions": {
     "ask": [
+      "Bash(git commit*)",
       "Edit(CHANGELOG.md)",
       "Write(CHANGELOG.md)",
       "Edit(pyproject.toml)",

CHANGELOG.md

@@ -1,5 +1,56 @@
 # Changelog
 
+## [3.4.1] - 2026-08-05
+
+- Backup: each volume is copied twice into the same destination — once hot,
+  once cold after the container is stopped — and rsync ran with *-b*, so
+  *--delete* renamed rather than removed a file the source had dropped between
+  the passes. Stopping a container is what makes the source drop files: a
+  graceful shutdown flushes and the format rolls its commit point. The
+  superseded file survived as *name~* beside the real one and was restored into
+  live data.
+- Backup: for an opaque payload that is stale bytes nobody reads; for a format
+  that enumerates its own directory it is corruption. Lucene resolves the
+  current commit by parsing every file starting with *segments* as a radix-36
+  generation, so a restored *segments_3~* leaves the shard store unreadable and
+  the primary at *NO_VALID_SHARD_COPY*. With *.security-7* unallocatable the
+  reserved *elastic* user has no password hash, every probe answers 401 and the
+  container never turns healthy.
+- Backup: *--link-dest* already provides the incrementals and nothing reads the
+  twins — the restore path is an unfiltered *rsync -avv --delete* into the live
+  volume. Dropping *-b* leaves the predecessor generation byte-identical, keeps
+  the hardlinks intact and makes generations smaller, never larger.
+- Tests: the absence of *--backup* is asserted on the rsync invocation.
+
+## [3.4.0] - 2026-08-02
+
+- Backup: *-a* implies *-D*, so a generation was written with
+  *--devices --specials* and rsync recreated every unix socket and fifo found in
+  a volume. Where the backup root is an nfs-ganesha export, ganesha accepts the
+  socket on write but cannot serve it back, and the remote pull's sender then
+  fails with *readdir* / *readlink_stat* "Invalid argument (22)" and exits 23 —
+  deterministically, for every retry. *--no-D* keeps them out of the generation.
+- Backup: nothing restorable is lost. Sockets and fifos are recreated by the
+  daemons that own them, and the postfix queue itself — *incoming*, *active*,
+  *deferred*, *hold*, *maildrop* — is unaffected, so accepted-but-undelivered
+  mail stays in the backup. Device nodes go too; the only volume that could hold
+  them is a nested docker data root, which does not belong in a backup anyway.
+- Tests: the flag is asserted on the rsync invocation.
+
+## [3.3.0] - 2026-08-02
+
+- Backup: *--volumes-no-backup-required* excludes a volume by name.
+  *--images-no-backup-required* resolves through *volume_is_fully_ignored*,
+  which skips a volume only when every container using it is ignored — a
+  container holding a derived tree beside state that must be kept cannot
+  express the exclusion at all. A docker-in-docker data root is exactly that
+  shape, and excluding by image would drop all three of its volumes.
+- Backup: the name check runs before *containers_using_volume*, so an excluded
+  volume costs no docker inspection and the decision does not depend on which
+  containers exist when the run starts.
+- Tests: two volumes off one container, asserting the sibling survives — the
+  property the image lever cannot provide — as unit and end-to-end.
+
 ## [3.2.2] - 2026-07-31
 
 - Backup: the btrfs snapshot is carved inside its subject, as

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "backup-docker-to-local"
-version = "3.2.2"
+version = "3.4.1"
 description = "Backup Docker volumes to local with rsync and optional DB dumps."
 readme = "README.md"
 requires-python = ">=3.9"

src/baudolo/backup/app.py

@@ -54,6 +54,14 @@ def main() -> int:
 
         for volume_name in docker_volume_names():
             print(f"Start backup routine for volume: {volume_name}", flush=True)
+
+            if volume_name in args.volumes_no_backup_required:
+                print(
+                    f"Skipping volume '{volume_name}' entirely (declared no-backup).",
+                    flush=True,
+                )
+                continue
+
             containers = containers_using_volume(volume_name)
 
             if volume_is_fully_ignored(containers, args.images_no_backup_required):

src/baudolo/backup/cli.py

@@ -68,6 +68,13 @@ def parse_args() -> argparse.Namespace:
         help="Exact image references (repo:tag, incl. any registry prefix) for which no backup should be performed",
     )
 
+    p.add_argument(
+        "--volumes-no-backup-required",
+        nargs="+",
+        default=[],
+        help="Exact volume names that are never backed up, whatever containers use them. For derived trees a restore cannot reproduce, above all a nested docker data root",
+    )
+
     p.add_argument(
         "--everything",
         action="store_true",
@@ -93,7 +100,9 @@ def parse_args() -> argparse.Namespace:
     if bool(args.snapshot) != bool(args.snapshot_subject):
         p.error("--snapshot and --snapshot-subject must be given together")
     if args.snapshot and args.shutdown:
-        p.error("--shutdown is meaningless with --snapshot: containers are never stopped")
+        p.error(
+            "--shutdown is meaningless with --snapshot: containers are never stopped"
+        )
     if args.snapshot and args.hard_restart_projects:
         p.error(
             "--hard-restart-projects is meaningless with --snapshot: the flag exists "

src/baudolo/backup/volume.py

@@ -49,7 +49,10 @@ def backup_volume(
     link_dest = f"--link-dest='{last}'" if last else ""
     verify = "--checksum " if authoritative else ""
 
-    cmd = f"rsync -abP --delete --delete-excluded {verify}{link_dest} {source} {dest}"
+    cmd = (
+        f"rsync -aP --no-D --delete --delete-excluded "
+        f"{verify}{link_dest} {source} {dest}"
+    )
 
     try:
         execute_shell_command(cmd)

tests/e2e/helpers/fixtures.py

@@ -1,4 +1,5 @@
 """Fixtures and paths the e2e suite builds its scenarios from."""
+
 from __future__ import annotations
 
 import shutil

tests/e2e/helpers/process.py

@@ -1,4 +1,5 @@
 """Process, docker and readiness helpers for the e2e suite."""
+
 from __future__ import annotations
 
 import subprocess

tests/e2e/snapshot_db_driver.py

@@ -25,7 +25,9 @@ GENERATION = f"{VERSIONS}/20260731"
 def shell(command: str) -> list[str]:
     proc = subprocess.run(command, shell=True, capture_output=True, text=True)
     if proc.returncode != 0:
-        raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
+        raise SnapshotError(
+            f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
+        )
     return proc.stdout.splitlines()
 
 

tests/e2e/snapshot_driver.py

@@ -22,7 +22,9 @@ EXPECT = sys.argv[3]
 def shell(command: str) -> list[str]:
     proc = subprocess.run(command, shell=True, capture_output=True, text=True)
     if proc.returncode != 0:
-        raise SnapshotError(f"{command} exited {proc.returncode}: {proc.stderr.strip()}")
+        raise SnapshotError(
+            f"{command} exited {proc.returncode}: {proc.stderr.strip()}"
+        )
     return proc.stdout.splitlines()
 
 
@@ -55,5 +57,8 @@ with volume_snapshot(KIND, SUBJECT, "e2e", run=shell) as resolve:
 
     root = Path(resolve(SUBJECT))
 
-check("the snapshot is removed afterwards", not root.exists() or not (root / "volumes").exists())
+check(
+    "the snapshot is removed afterwards",
+    not root.exists() or not (root / "volumes").exists(),
+)
 print("ALL OK", flush=True)

tests/e2e/test_e2e_snapshot.py

@@ -29,7 +29,7 @@ LOOP_FS = {
     "ext4": "mkdir -p /subject/docker",
 }
 # A container carries no /lib/modules, so modprobe fails even on a loaded module.
-ZFS_READY = '{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]'
+ZFS_READY = "{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]"
 
 
 def mount_script(fstype: str) -> str:
@@ -51,7 +51,13 @@ def zfs_usable() -> bool:
     """Whether this host's kernel can serve zfs to a privileged container."""
     proc = run(
         [
-            "docker", "run", "--rm", "--privileged", IMAGE, "sh", "-lc",
+            "docker",
+            "run",
+            "--rm",
+            "--privileged",
+            IMAGE,
+            "sh",
+            "-lc",
             f"apk add -q zfs >/dev/null 2>&1 && {ZFS_READY}",
         ],
         capture=True,
@@ -87,11 +93,20 @@ def drive(fstype: str, kind: str, expect: str) -> str:
     try:
         proc = run(
             [
-                "docker", "run", "--rm", "--privileged",
-                "--name", staged.name,
-                "-v", f"{staged / 'src'}:/src:ro",
-                "-v", f"{staged / 'driver.py'}:/driver.py:ro",
-                IMAGE, "sh", "-lc", script,
+                "docker",
+                "run",
+                "--rm",
+                "--privileged",
+                "--name",
+                staged.name,
+                "-v",
+                f"{staged / 'src'}:/src:ro",
+                "-v",
+                f"{staged / 'driver.py'}:/driver.py:ro",
+                IMAGE,
+                "sh",
+                "-lc",
+                script,
             ],
             capture=True,
             check=False,
@@ -99,7 +114,9 @@ def drive(fstype: str, kind: str, expect: str) -> str:
     finally:
         shutil.rmtree(staged, ignore_errors=True)
     if proc.returncode != 0:
-        raise AssertionError(f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}")
+        raise AssertionError(
+            f"{fstype}/{kind} driver failed:\n{proc.stdout}\n{proc.stderr}"
+        )
     return proc.stdout
 
 
@@ -125,7 +142,9 @@ class TestE2ESnapshot(unittest.TestCase):
                     "E2E_REQUIRE_FILESYSTEMS demands zfs, but this kernel provides no "
                     "zfs module; load it before running the suite"
                 )
-            self.skipTest("this kernel provides no zfs module, so no pool can be created")
+            self.skipTest(
+                "this kernel provides no zfs module, so no pool can be created"
+            )
         self.assert_freezes("zfs")
 
     def test_ext4_has_no_snapshot_and_says_so(self) -> None:

tests/e2e/test_e2e_snapshot_db.py

@@ -68,11 +68,20 @@ class TestE2ESnapshotDatabase(unittest.TestCase):
         try:
             proc = run(
                 [
-                    "docker", "run", "--rm", "--privileged",
-                    "--name", staged.name,
-                    "-v", f"{staged / 'src'}:/src:ro",
-                    "-v", f"{staged / 'driver.py'}:/driver.py:ro",
-                    IMAGE, "sh", "-lc", SCRIPT,
+                    "docker",
+                    "run",
+                    "--rm",
+                    "--privileged",
+                    "--name",
+                    staged.name,
+                    "-v",
+                    f"{staged / 'src'}:/src:ro",
+                    "-v",
+                    f"{staged / 'driver.py'}:/driver.py:ro",
+                    IMAGE,
+                    "sh",
+                    "-lc",
+                    SCRIPT,
                 ],
                 capture=True,
                 check=False,

tests/e2e/test_e2e_volumes_no_backup_required_early_skip.py

@@ -0,0 +1,125 @@
+import unittest
+
+from .helpers import (
+    backup_path,
+    cleanup_docker,
+    create_minimal_compose_dir,
+    ensure_empty_dir,
+    latest_version_dir,
+    require_docker,
+    run,
+    unique,
+    write_databases_csv,
+)
+
+
+class TestE2EVolumesNoBackupRequiredEarlySkip(unittest.TestCase):
+    """Both volumes hang off the same container, so an image-level exclusion
+    could only drop both. Only the named one may disappear."""
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+
+        cls.prefix = unique("baudolo-e2e-early-skip-no-backup-volume")
+        cls.backups_dir = f"/tmp/{cls.prefix}/Backups"
+        ensure_empty_dir(cls.backups_dir)
+
+        cls.compose_dir = create_minimal_compose_dir(f"/tmp/{cls.prefix}")
+        cls.repo_name = cls.prefix
+
+        cls.container = f"{cls.prefix}-app"
+        cls.excluded_volume = f"{cls.prefix}-derived-vol"
+        cls.kept_volume = f"{cls.prefix}-state-vol"
+
+        cls.containers = [cls.container]
+        cls.volumes = [cls.excluded_volume, cls.kept_volume]
+
+        run(["docker", "volume", "create", cls.excluded_volume])
+        run(["docker", "volume", "create", cls.kept_volume])
+
+        run(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "-v",
+                f"{cls.excluded_volume}:/derived",
+                "-v",
+                f"{cls.kept_volume}:/state",
+                "alpine:3.20",
+                "sh",
+                "-lc",
+                "echo derived > /derived/derived.txt && echo state > /state/state.txt",
+            ]
+        )
+
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.container,
+                "-v",
+                f"{cls.excluded_volume}:/derived",
+                "-v",
+                f"{cls.kept_volume}:/state",
+                "alpine:3.20",
+                "sleep",
+                "600",
+            ]
+        )
+
+        cls.databases_csv = f"/tmp/{cls.prefix}/databases.csv"
+        write_databases_csv(cls.databases_csv, [])
+
+        cmd = [
+            "baudolo",
+            "--compose-dir",
+            cls.compose_dir,
+            "--repo-name",
+            cls.repo_name,
+            "--databases-csv",
+            cls.databases_csv,
+            "--backups-dir",
+            cls.backups_dir,
+            "--images-no-stop-required",
+            "alpine:3.20",
+            "--volumes-no-backup-required",
+            cls.excluded_volume,
+        ]
+        cp = run(cmd, capture=True, check=True)
+        cls.stdout = cp.stdout or ""
+        cls.stderr = cp.stderr or ""
+
+        cls.hash, cls.version = latest_version_dir(cls.backups_dir, cls.repo_name)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_excluded_volume_has_no_backup_directory_at_all(self) -> None:
+        p = backup_path(
+            self.backups_dir,
+            self.repo_name,
+            self.version,
+            self.excluded_volume,
+        )
+        self.assertFalse(
+            p.exists(),
+            f"Expected NO backup directory for the excluded volume, but found: {p}",
+        )
+
+    def test_sibling_volume_of_the_same_container_is_still_backed_up(self) -> None:
+        p = (
+            backup_path(
+                self.backups_dir,
+                self.repo_name,
+                self.version,
+                self.kept_volume,
+            )
+            / "files"
+            / "state.txt"
+        )
+        self.assertTrue(p.is_file(), f"Expected backed up file at: {p}")

tests/unit/backup/test_app_snapshot.py

@@ -12,6 +12,7 @@ from baudolo.backup.snapshot import volume_snapshot
 def stubbed_snapshot(kind: str, subject: str, tag: str):
     return volume_snapshot(kind, subject, tag, run=lambda command: [])
 
+
 ARGV = [
     "baudolo",
     "--compose-dir",

tests/unit/backup/test_app_volumes_no_backup_required.py

@@ -0,0 +1,82 @@
+"""Contract of --volumes-no-backup-required: exclusion is per volume name,
+independent of which containers use it."""
+
+from __future__ import annotations
+
+import unittest
+from unittest import mock
+
+from baudolo.backup import app
+
+ARGV = [
+    "baudolo",
+    "--compose-dir",
+    "/compose",
+    "--backups-dir",
+    "/backups",
+    "--volumes-no-backup-required",
+    "derived",
+]
+
+
+def drive() -> tuple[list[str], list[str], list[str]]:
+    backed_up: list[str] = []
+    created: list[str] = []
+    inspected: list[str] = []
+
+    def record_backup(versions_dir, volume_name, volume_dir, *, authoritative, source):
+        backed_up.append(volume_name)
+
+    with (
+        mock.patch("sys.argv", ARGV),
+        mock.patch.object(app, "get_machine_id", return_value="machine"),
+        mock.patch.object(app, "create_version_directory", return_value="/gen"),
+        mock.patch.object(
+            app,
+            "create_volume_directory",
+            side_effect=lambda _version_dir, name: created.append(name) or "/gen/vol",
+        ),
+        mock.patch.object(app, "load_databases_df", return_value=None),
+        mock.patch.object(
+            app, "docker_volume_names", return_value=["derived", "state"]
+        ),
+        mock.patch.object(
+            app,
+            "containers_using_volume",
+            side_effect=lambda name: inspected.append(name) or ["app"],
+        ),
+        mock.patch.object(app, "volume_is_fully_ignored", return_value=False),
+        mock.patch.object(app, "backup_dumps_for_volume", return_value=(False, False)),
+        mock.patch.object(app, "get_storage_path", return_value="/data/"),
+        mock.patch.object(app, "stamp_directory"),
+        mock.patch.object(app, "handle_docker_compose_services"),
+        mock.patch.object(app.os.path, "isdir", return_value=True),
+        mock.patch.object(app, "backup_volume", side_effect=record_backup),
+        mock.patch.object(app, "filter_stoppable", return_value=[]),
+        mock.patch.object(app, "requires_stop", return_value=False),
+        mock.patch.object(app, "change_containers_status"),
+    ):
+        app.main()
+    return backed_up, created, inspected
+
+
+class TestVolumesNoBackupRequired(unittest.TestCase):
+    def test_the_named_volume_is_never_backed_up(self) -> None:
+        backed_up, _created, _inspected = drive()
+        self.assertNotIn("derived", backed_up)
+
+    def test_a_sibling_volume_of_the_same_container_survives(self) -> None:
+        backed_up, _created, _inspected = drive()
+        self.assertEqual(backed_up, ["state"])
+
+    def test_no_generation_directory_is_created_for_it(self) -> None:
+        _backed_up, created, _inspected = drive()
+        self.assertEqual(created, ["state"])
+
+    def test_the_skip_precedes_the_container_inspection(self) -> None:
+        _backed_up, _created, inspected = drive()
+        self.assertEqual(inspected, ["state"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/backup/test_cli.py

@@ -39,7 +39,9 @@ class TestSnapshotFlags(unittest.TestCase):
             parse("--snapshot", "ext4", "--snapshot-subject", "/var/lib/docker")
 
     def test_zfs_is_accepted(self) -> None:
-        self.assertEqual(parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs")
+        self.assertEqual(
+            parse("--snapshot", "zfs", "--snapshot-subject", "/d").snapshot, "zfs"
+        )
 
     def test_shutdown_is_rejected_because_nothing_is_stopped(self) -> None:
         with self.assertRaises(SystemExit):

tests/unit/backup/test_snapshot.py

@@ -35,11 +35,15 @@ class TestBtrfs(unittest.TestCase):
         run = Runner()
         with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run):
             pass
-        self.assertEqual(run.calls[-1], "btrfs subvolume delete /var/lib/docker/.baudolo-20260731")
+        self.assertEqual(
+            run.calls[-1], "btrfs subvolume delete /var/lib/docker/.baudolo-20260731"
+        )
 
     def test_it_maps_a_volume_path_into_the_snapshot(self) -> None:
         run = Runner()
-        with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
+        with volume_snapshot(
+            "btrfs", "/var/lib/docker", "20260731", run=run
+        ) as resolve:
             self.assertEqual(
                 resolve("/var/lib/docker/volumes/postgres_data/_data"),
                 "/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data",
@@ -47,7 +51,9 @@ class TestBtrfs(unittest.TestCase):
 
     def test_it_keeps_the_trailing_slash_rsync_reads_as_contents(self) -> None:
         run = Runner()
-        with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
+        with volume_snapshot(
+            "btrfs", "/var/lib/docker", "20260731", run=run
+        ) as resolve:
             self.assertEqual(
                 resolve("/var/lib/docker/volumes/postgres_data/_data/"),
                 "/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data/",
@@ -102,14 +108,20 @@ class TestRejections(unittest.TestCase):
 
     def test_a_path_outside_the_subject_is_rejected(self) -> None:
         run = Runner()
-        with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
+        with volume_snapshot(
+            "btrfs", "/var/lib/docker", "20260731", run=run
+        ) as resolve:
             with self.assertRaises(SnapshotError):
                 resolve("/etc/passwd")
 
     def test_the_subject_itself_resolves_to_the_snapshot_root(self) -> None:
         run = Runner()
-        with volume_snapshot("btrfs", "/var/lib/docker", "20260731", run=run) as resolve:
-            self.assertEqual(resolve("/var/lib/docker"), "/var/lib/docker/.baudolo-20260731")
+        with volume_snapshot(
+            "btrfs", "/var/lib/docker", "20260731", run=run
+        ) as resolve:
+            self.assertEqual(
+                resolve("/var/lib/docker"), "/var/lib/docker/.baudolo-20260731"
+            )
 
 
 class Busy(Runner):

tests/unit/backup/test_volume.py

@@ -43,6 +43,14 @@ class TestBackupVolume(unittest.TestCase):
     def test_it_always_deletes_what_the_source_no_longer_has(self) -> None:
         self.assertIn("--delete", self.copy())
 
+    def test_it_carries_no_kernel_objects_into_a_generation(self) -> None:
+        self.assertIn("--no-D", self.copy())
+
+    def test_it_keeps_no_twin_of_what_the_second_pass_replaces(self) -> None:
+        command = self.copy(authoritative=True)
+        self.assertIn("rsync -aP ", command)
+        self.assertNotIn("--backup", command)
+
     def test_it_creates_the_destination(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             dest = Path(tmp) / "gen" / "demo"