fix(backup): re-raise inspect failures for containers that still exist

Treating every failed swarm-task inspect as skippable opened a false-green
window: a transient inspect failure on a still-running, non-whitelisted
container skipped the stop and backed the volume up hot while the run
reported success. Re-check whether the container is still listed; only a
genuinely vanished container skips, an existing one re-raises so a broken
daemon keeps failing the run loudly. Covered by unit tests for both paths.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-11 09:28:21 +02:00
parent c949f2c5cf
commit b9a8b391f0
2 changed files with 21 additions and 2 deletions

View File

@@ -29,13 +29,20 @@ def is_swarm_task(container: str) -> bool:
manually: the orchestrator replaces the stopped task and a later manually: the orchestrator replaces the stopped task and a later
`docker start` fails on the detached overlay network. A container that `docker start` fails on the detached overlay network. A container that
vanished between listing and inspect (--rm one-shots, task-history GC) vanished between listing and inspect (--rm one-shots, task-history GC)
counts as not stoppable instead of aborting the whole backup run.""" counts as not stoppable instead of aborting the whole backup run; if the
container still exists the inspect failure re-raises, so a broken daemon
keeps failing the run loudly instead of silently skipping the stop."""
try: try:
out = execute_shell_command( out = execute_shell_command(
"docker inspect --format " "docker inspect --format "
f"'{{{{index .Config.Labels \"com.docker.swarm.task.id\"}}}}' {container}" f"'{{{{index .Config.Labels \"com.docker.swarm.task.id\"}}}}' {container}"
) )
except BackupException: except BackupException:
still_listed = execute_shell_command(
f"docker ps -a --filter name=^{container}$ --format '{{{{.Names}}}}'"
)
if still_listed and still_listed[0].strip():
raise
return True return True
return bool(out and out[0].strip()) return bool(out and out[0].strip())

View File

@@ -21,7 +21,7 @@ class TestIsSwarmTask(unittest.TestCase):
@patch.object( @patch.object(
docker_mod, docker_mod,
"execute_shell_command", "execute_shell_command",
side_effect=BackupException("gone"), side_effect=[BackupException("gone"), []],
) )
def test_vanished_container_counts_as_not_stoppable(self, _mock) -> None: def test_vanished_container_counts_as_not_stoppable(self, _mock) -> None:
# A container removed between listing and inspect must not abort the # A container removed between listing and inspect must not abort the
@@ -29,6 +29,18 @@ class TestIsSwarmTask(unittest.TestCase):
# stop/start and image-inspect path. # stop/start and image-inspect path.
self.assertTrue(docker_mod.is_swarm_task("gone-container")) self.assertTrue(docker_mod.is_swarm_task("gone-container"))
@patch.object(
docker_mod,
"execute_shell_command",
side_effect=[BackupException("daemon hiccup"), ["still-here"]],
)
def test_inspect_failure_on_existing_container_still_fails(self, _mock) -> None:
# If the container still exists, an inspect failure must keep failing
# the run: silently skipping the stop would back up a hot volume and
# report green without the stop guarantee.
with self.assertRaises(BackupException):
docker_mod.is_swarm_task("still-here")
class TestFilterStoppable(unittest.TestCase): class TestFilterStoppable(unittest.TestCase):
@patch.object(docker_mod, "is_swarm_task", side_effect=[False, True, False]) @patch.object(docker_mod, "is_swarm_task", side_effect=[False, True, False])