From cd21f1fa6791118632ed339f4b9cea23d10641b6 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 5 Aug 2026 12:43:08 +0200 Subject: [PATCH] fix(backup): keep no twin of what the cold pass replaces Each volume is copied twice into the same destination: once hot with the container running, once cold after it is stopped (backup/app.py:127-131). rsync ran with -b, so --delete did not remove a file the source had dropped between the passes - it renamed it. Stopping a container is exactly what makes the source drop files: a graceful shutdown flushes, and the format rolls its commit point. For an opaque payload the twin is stale bytes nobody reads. For a format that enumerates its own directory it is corruption. Lucene resolves the current commit by parsing every file starting with segments as a radix-36 generation, so a restored segments_3~ raises NumberFormatException, the shard store cannot be read, and the primary is left NO_VALID_SHARD_COPY. With .security-7 unallocatable the reserved elastic user has no password hash, every probe gets HTTP 401, and the container never turns healthy. Seen in infinito-nexus-core CI run 30963648828: the restore drill waited 1200s on elasticsearch while all 29 other containers came back healthy; the generation held segments_3~ next to segments_4 in all three index directories. A run two days earlier passed the same drill because that stack was idle and nothing rolled between the passes - which is why this reads as flaky rather than broken. Reproduced with real rsync in all three shapes: two passes into the same destination with -b leave segments_3~ beside segments_4, without -b only segments_4 survives, and a single pass keeps segments_3. Measured on the same fixtures, dropping -b leaves the predecessor generation byte-identical and keeps the --link-dest hardlinks intact, so the incremental scheme is unaffected; generations get smaller, never larger. --link-dest already provides the cheap incrementals. -b contributed nothing on top of it but the twins, and no caller reads them: the restore path is an unfiltered rsync -avv --delete into the live volume (restore/files.py:35). Co-Authored-By: Claude Opus 5 (1M context) --- src/baudolo/backup/volume.py | 2 +- tests/unit/backup/test_volume.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/baudolo/backup/volume.py b/src/baudolo/backup/volume.py index 1d4fddd..e9d4285 100644 --- a/src/baudolo/backup/volume.py +++ b/src/baudolo/backup/volume.py @@ -50,7 +50,7 @@ def backup_volume( verify = "--checksum " if authoritative else "" cmd = ( - f"rsync -abP --no-D --delete --delete-excluded " + f"rsync -aP --no-D --delete --delete-excluded " f"{verify}{link_dest} {source} {dest}" ) diff --git a/tests/unit/backup/test_volume.py b/tests/unit/backup/test_volume.py index 88c1022..80bb7cd 100644 --- a/tests/unit/backup/test_volume.py +++ b/tests/unit/backup/test_volume.py @@ -46,6 +46,11 @@ class TestBackupVolume(unittest.TestCase): def test_it_carries_no_kernel_objects_into_a_generation(self) -> None: self.assertIn("--no-D", self.copy()) + def test_it_keeps_no_twin_of_what_the_second_pass_replaces(self) -> None: + command = self.copy(authoritative=True) + self.assertIn("rsync -aP ", command) + self.assertNotIn("--backup", command) + def test_it_creates_the_destination(self) -> None: with tempfile.TemporaryDirectory() as tmp: dest = Path(tmp) / "gen" / "demo"