mirror of
https://github.com/kevinveenbirkenbach/docker-volume-backup.git
synced 2026-08-24 14:54:32 +00:00
Compare commits
4 Commits
v6.0.0
...
44b1f16f7c
| Author | SHA1 | Date | |
|---|---|---|---|
| 44b1f16f7c | |||
| 7f51748486 | |||
| e0f89c86ec | |||
| f97efb10c4 |
50
CHANGELOG.md
50
CHANGELOG.md
@@ -1,5 +1,55 @@
|
||||
# Changelog
|
||||
|
||||
## [7.0.1] - 2026-08-18
|
||||
|
||||
- Restore: *--empty* no longer aborts on a database that carries an extension.
|
||||
The pre-clean picked its candidates by owner, on the stated assumption that
|
||||
extension members are superuser-owned and would therefore never be selected.
|
||||
That holds only when a superuser installed the extension: a role that
|
||||
installs one itself owns its functions, so they were listed for a one-by-one
|
||||
*DROP* that postgres refuses — *cannot drop function
|
||||
vector_in(cstring,oid,integer) because extension vector requires it*. Under
|
||||
*ON_ERROR_STOP* that ends the whole restore, which is how a generation of an
|
||||
application declaring the *vector* extension became unreplayable. Membership
|
||||
now comes from *pg_depend* rather than from ownership; each branch carries
|
||||
its oid and classid so one *NOT EXISTS* covers all seven instead of seven
|
||||
separate predicates, and the schema branch is guarded too because an
|
||||
extension can own a schema. Skipping the members suffices — the dump's
|
||||
*CREATE EXTENSION IF NOT EXISTS* finds the surviving extension either way.
|
||||
|
||||
## [7.0.0] - 2026-08-18
|
||||
|
||||
Breaking:
|
||||
- Backup: *mariadb* and *mysql* join the suffix tokens, so a container named
|
||||
*<app>-mariadb* or *<app>-mysql* now resolves to the instance *<app>* instead
|
||||
of to its own name. A *databases.csv* keyed on the full container name has to
|
||||
move to the application name, or name the container in
|
||||
*--database-containers*. This narrows what 6.0.0 broke rather than widening
|
||||
it: a container named exactly *postgres*, *mariadb*, *mysql*, *db* or
|
||||
*database* resolves again without any declaration, which is the shape a
|
||||
compose file writes as *container_name: postgres* and the most common
|
||||
configuration there is.
|
||||
|
||||
Fixed:
|
||||
- Backup: a container named exactly after its engine is dumped again. The
|
||||
suffix match needs a hyphen or underscore in front of the token, which a bare
|
||||
name does not carry, so 6.0.0 resolved *container_name: postgres* to nothing
|
||||
and stopped dumping it without saying so. *ENGINE_NAMES* now states the set
|
||||
once and serves both readings — carried as a suffix it makes the rest the
|
||||
instance, being one outright makes the container its own instance.
|
||||
- Backup: a central MariaDB under swarm is dumped for the first time. Swarm
|
||||
names its task *mariadb_mariadb.1.<id>*, which matches neither the static
|
||||
*mariadb* passed through *--database-containers* nor any token the suffix
|
||||
match knew, since *_mariadb* is not *_db*. The database was silently absent
|
||||
from every swarm backup this tool has ever written, before 6.0.0 as well.
|
||||
- Backup: an application container is no longer recorded as a database.
|
||||
*container_engine* recognises an engine by its client tools, which an
|
||||
application image frequently ships, so refusing its dump alone would have
|
||||
written the volume to the manifest as *database: true, dumped: false* — the
|
||||
exact shape a restore drill reads as a database that was missed. Without an
|
||||
instance there is no database to record, and the volume is a file backup like
|
||||
any other.
|
||||
|
||||
## [6.0.0] - 2026-08-18
|
||||
|
||||
Breaking:
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "backup-docker-to-local"
|
||||
version = "6.0.0"
|
||||
version = "7.0.1"
|
||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
||||
@@ -16,13 +16,18 @@ if TYPE_CHECKING:
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ENGINE_NAMES = ("database", "postgres", "mariadb", "mysql", "db")
|
||||
_SUFFIX_RE = re.compile(rf"(_|-)({'|'.join(ENGINE_NAMES)})")
|
||||
|
||||
|
||||
def get_instance(container: str, database_containers: list[str]) -> str | None:
|
||||
"""The databases.csv instance a container serves, or None for no database.
|
||||
|
||||
A declared container is its own instance. Every other name is normalised by
|
||||
stripping a database suffix token, which maps both `<app>-database` from
|
||||
compose and `<app>_database.1.<task>` from swarm onto the same instance.
|
||||
A declared container is its own instance. Every other name is read against
|
||||
ENGINE_NAMES: carrying one as a suffix makes the rest the instance, which
|
||||
maps `<app>-database` from compose and `<app>_database.1.<task>` from swarm
|
||||
onto the same one; being one outright makes the container its own instance,
|
||||
the shape a compose file writes as `container_name: postgres`.
|
||||
|
||||
Args:
|
||||
container: the running container's name.
|
||||
@@ -30,14 +35,16 @@ def get_instance(container: str, database_containers: list[str]) -> str | None:
|
||||
declared engines whatever they are called.
|
||||
|
||||
Returns:
|
||||
The instance name, or None when the name carries no database token: an
|
||||
application container is not an engine, even when it ships the client
|
||||
tools that would let a dump command start.
|
||||
The instance name, or None when the name neither carries nor is an
|
||||
engine name: an application container is not an engine, even when it
|
||||
ships the client tools that would let a dump command start.
|
||||
"""
|
||||
if container in database_containers:
|
||||
return container
|
||||
parts = re.split(r"(_|-)(database|db|postgres)", container)
|
||||
return parts[0] if len(parts) > 1 else None
|
||||
parts = _SUFFIX_RE.split(container)
|
||||
if len(parts) > 1:
|
||||
return parts[0]
|
||||
return container if container in ENGINE_NAMES else None
|
||||
|
||||
|
||||
def fallback_pg_dumpall(
|
||||
|
||||
@@ -10,7 +10,7 @@ from pandas.errors import EmptyDataError
|
||||
|
||||
from baudolo.databases import COLUMNS, DELIMITER
|
||||
|
||||
from .db import backup_database
|
||||
from .db import backup_database, get_instance
|
||||
from .docker import has_tool, image_id
|
||||
|
||||
DUMP_TOOLS: tuple[tuple[str, str], ...] = (
|
||||
@@ -76,6 +76,8 @@ def backup_mariadb_or_postgres(
|
||||
engine = container_engine(container)
|
||||
if engine is None:
|
||||
return VolumeOutcome(database=False, dumped=False)
|
||||
if get_instance(container, database_containers) is None:
|
||||
return VolumeOutcome(database=False, dumped=False)
|
||||
db_type, dump_tool = engine
|
||||
dumped = backup_database(
|
||||
container=container,
|
||||
|
||||
@@ -3,8 +3,12 @@
|
||||
-- would run every DROP in one transaction and exhaust max_locks_per_transaction on
|
||||
-- large schemas (e.g. gitlab). Also drops user-owned non-public schemas so a dump
|
||||
-- that CREATE SCHEMAs (e.g. discourse's discourse_functions) does not fail on an
|
||||
-- already-existing schema. Extension members (pg_trgm's set_limit) are
|
||||
-- superuser-owned; IF EXISTS absorbs the CASCADE fallout.
|
||||
-- already-existing schema. Objects belonging to an extension are skipped: postgres
|
||||
-- refuses to drop them one by one ("cannot drop function vector_in(...) because
|
||||
-- extension vector requires it"), and the dump's CREATE EXTENSION IF NOT EXISTS
|
||||
-- finds the surviving extension either way. Owning them is not enough to make them
|
||||
-- droppable - an extension a role installed itself is owned by that role, so the
|
||||
-- owner filter alone lets pgvector's members through.
|
||||
SELECT format('DROP %s IF EXISTS public.%s CASCADE', obj.type, obj.name)
|
||||
FROM (
|
||||
SELECT format('%I', c.relname) AS name,
|
||||
@@ -13,7 +17,8 @@ SELECT format('DROP %s IF EXISTS public.%s CASCADE', obj.type, obj.name)
|
||||
WHEN 'm' THEN 'MATERIALIZED VIEW'
|
||||
WHEN 'f' THEN 'FOREIGN TABLE'
|
||||
ELSE 'TABLE'
|
||||
END AS type
|
||||
END AS type,
|
||||
c.oid AS objid, 'pg_class'::regclass AS classid
|
||||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p', 'v', 'm', 'f')
|
||||
AND pg_get_userbyid(c.relowner) = current_user
|
||||
@@ -21,17 +26,20 @@ SELECT format('DROP %s IF EXISTS public.%s CASCADE', obj.type, obj.name)
|
||||
-- Overloaded functions share a proname; DROP needs the identity
|
||||
-- signature or psql aborts with "function name is not unique".
|
||||
SELECT format('%I(%s)', p.proname, pg_get_function_identity_arguments(p.oid)) AS name,
|
||||
CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END AS type
|
||||
CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END AS type,
|
||||
p.oid AS objid, 'pg_proc'::regclass AS classid
|
||||
FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
|
||||
WHERE n.nspname = 'public' AND p.prokind IN ('f', 'p', 'w')
|
||||
AND pg_get_userbyid(p.proowner) = current_user
|
||||
UNION ALL
|
||||
SELECT format('%I', c.relname) AS name, 'SEQUENCE' AS type
|
||||
SELECT format('%I', c.relname) AS name, 'SEQUENCE' AS type,
|
||||
c.oid AS objid, 'pg_class'::regclass AS classid
|
||||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relkind = 'S'
|
||||
AND pg_get_userbyid(c.relowner) = current_user
|
||||
UNION ALL
|
||||
SELECT format('%I', t.typname) AS name, 'TYPE' AS type
|
||||
SELECT format('%I', t.typname) AS name, 'TYPE' AS type,
|
||||
t.oid AS objid, 'pg_type'::regclass AS classid
|
||||
FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND pg_get_userbyid(t.typowner) = current_user
|
||||
@@ -40,25 +48,36 @@ SELECT format('DROP %s IF EXISTS public.%s CASCADE', obj.type, obj.name)
|
||||
SELECT 1 FROM pg_class c2
|
||||
WHERE c2.oid = t.typrelid AND c2.relkind = 'c')))
|
||||
UNION ALL
|
||||
SELECT format('%I', col.collname) AS name, 'COLLATION' AS type
|
||||
SELECT format('%I', col.collname) AS name, 'COLLATION' AS type,
|
||||
col.oid AS objid, 'pg_collation'::regclass AS classid
|
||||
FROM pg_collation col JOIN pg_namespace n ON n.oid = col.collnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND pg_get_userbyid(col.collowner) = current_user
|
||||
UNION ALL
|
||||
SELECT format('%I', ts.cfgname) AS name, 'TEXT SEARCH CONFIGURATION' AS type
|
||||
SELECT format('%I', ts.cfgname) AS name, 'TEXT SEARCH CONFIGURATION' AS type,
|
||||
ts.oid AS objid, 'pg_ts_config'::regclass AS classid
|
||||
FROM pg_ts_config ts JOIN pg_namespace n ON n.oid = ts.cfgnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND pg_get_userbyid(ts.cfgowner) = current_user
|
||||
UNION ALL
|
||||
SELECT format('%I', d.dictname) AS name, 'TEXT SEARCH DICTIONARY' AS type
|
||||
SELECT format('%I', d.dictname) AS name, 'TEXT SEARCH DICTIONARY' AS type,
|
||||
d.oid AS objid, 'pg_ts_dict'::regclass AS classid
|
||||
FROM pg_ts_dict d JOIN pg_namespace n ON n.oid = d.dictnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND pg_get_userbyid(d.dictowner) = current_user
|
||||
) obj
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM pg_depend dep
|
||||
WHERE dep.classid = obj.classid AND dep.objid = obj.objid
|
||||
AND dep.deptype = 'e')
|
||||
UNION ALL
|
||||
SELECT format('DROP SCHEMA IF EXISTS %I CASCADE', n.nspname)
|
||||
FROM pg_namespace n
|
||||
WHERE NOT starts_with(n.nspname, 'pg_')
|
||||
AND n.nspname NOT IN ('public', 'information_schema')
|
||||
AND pg_get_userbyid(n.nspowner) = current_user
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM pg_depend dep
|
||||
WHERE dep.classid = 'pg_namespace'::regclass AND dep.objid = n.oid
|
||||
AND dep.deptype = 'e')
|
||||
\gexec
|
||||
|
||||
@@ -13,10 +13,11 @@ by its own launcher and ships pg_dumpall, so a dump command starts there and
|
||||
writes a file that looks like a backup and holds none of the data.
|
||||
"""
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from baudolo.generation import DUMP_SUFFIX, FILES_DIR, SQL_DIR
|
||||
from baudolo.generation import DUMP_SUFFIX, FILES_DIR, MANIFEST_FILE, SQL_DIR
|
||||
|
||||
from .helpers import (
|
||||
POSTGRES_DATA_DIR,
|
||||
@@ -158,6 +159,26 @@ class TestE2EAppContainerShipsClientTools(unittest.TestCase):
|
||||
self.assertTrue(marker.is_file(), f"expected a file backup at {marker}")
|
||||
self.assertIn(MARKER, marker.read_text(encoding="utf-8"))
|
||||
|
||||
def test_the_manifest_does_not_call_the_application_volume_a_database(self) -> None:
|
||||
manifest = json.loads(
|
||||
(self.volume_dir(self.app_volume).parent / MANIFEST_FILE).read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
entry = manifest["volumes"][self.app_volume]
|
||||
self.assertFalse(entry["database"], entry)
|
||||
self.assertFalse(entry["dumped"], entry)
|
||||
|
||||
def test_the_manifest_records_the_engine_volume_as_dumped(self) -> None:
|
||||
manifest = json.loads(
|
||||
(self.volume_dir(self.engine_volume).parent / MANIFEST_FILE).read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
entry = manifest["volumes"][self.engine_volume]
|
||||
self.assertTrue(entry["database"], entry)
|
||||
self.assertTrue(entry["dumped"], entry)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -24,6 +24,7 @@ from .helpers import (
|
||||
# is not unique") and english_stem_nostop reproduces taiga's text search
|
||||
# dictionary abort (duplicate pg_ts_dict_dictname_index).
|
||||
SCENARIO_SQL = (
|
||||
"CREATE EXTENSION pg_trgm;"
|
||||
"CREATE SCHEMA discourse_functions;"
|
||||
"CREATE TABLE discourse_functions.helper (id int);"
|
||||
"INSERT INTO discourse_functions.helper VALUES (1);"
|
||||
@@ -166,6 +167,13 @@ class TestE2EPostgresEmptyDropHard(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(self._scalar("SELECT public.f(41) + public.f();"), "42")
|
||||
|
||||
def test_the_extension_survived_the_preclean(self) -> None:
|
||||
self.assertEqual(
|
||||
self._scalar("SELECT count(*) FROM pg_extension WHERE extname='pg_trgm';"),
|
||||
"1",
|
||||
)
|
||||
self.assertEqual(self._scalar("SELECT similarity('abc','abc')::int;"), "1")
|
||||
|
||||
def test_text_search_dictionary_restored_once(self) -> None:
|
||||
self.assertEqual(
|
||||
self._scalar(
|
||||
|
||||
@@ -24,12 +24,21 @@ class TestDeclaredContainers(unittest.TestCase):
|
||||
get_instance("shop-database", ["shop-database"]), "shop-database"
|
||||
)
|
||||
|
||||
def test_an_undeclared_central_engine_resolves_to_nothing(self) -> None:
|
||||
"""`postgres-central` has no separator before its token, so nothing is
|
||||
stripped - a central engine has to be declared to be found."""
|
||||
def test_a_qualified_central_name_still_has_to_be_declared(self) -> None:
|
||||
self.assertIsNone(get_instance("postgres-central", []))
|
||||
|
||||
|
||||
class TestContainersNamedAfterTheirEngine(unittest.TestCase):
|
||||
def test_a_bare_engine_name_is_its_own_instance(self) -> None:
|
||||
for name in ("postgres", "mariadb", "mysql", "db", "database"):
|
||||
with self.subTest(container=name):
|
||||
self.assertEqual(get_instance(name, []), name)
|
||||
|
||||
def test_a_swarm_task_of_such_a_container_keeps_the_instance(self) -> None:
|
||||
self.assertEqual(get_instance("postgres_postgres.1.k3f9x2", []), "postgres")
|
||||
self.assertEqual(get_instance("mariadb_mariadb.1.k3f9x2", []), "mariadb")
|
||||
|
||||
|
||||
class TestDedicatedEngines(unittest.TestCase):
|
||||
def test_compose_names_the_container_with_a_hyphen(self) -> None:
|
||||
self.assertEqual(get_instance("discourse-database", []), "discourse")
|
||||
@@ -49,6 +58,10 @@ class TestDedicatedEngines(unittest.TestCase):
|
||||
def test_mariadb_uses_the_same_suffix(self) -> None:
|
||||
self.assertEqual(get_instance("matomo-database", []), "matomo")
|
||||
|
||||
def test_an_engine_named_suffix_is_stripped_too(self) -> None:
|
||||
self.assertEqual(get_instance("shop-mariadb", []), "shop")
|
||||
self.assertEqual(get_instance("shop-mysql", []), "shop")
|
||||
|
||||
|
||||
class TestApplicationContainers(unittest.TestCase):
|
||||
def test_a_bare_application_name_is_not_a_database(self) -> None:
|
||||
|
||||
Reference in New Issue
Block a user