diff --git a/plugins/lookup/nfs_tmpfs_mountpoints.py b/plugins/lookup/nfs_tmpfs_mountpoints.py new file mode 100644 index 0000000000..491e37bcd6 --- /dev/null +++ b/plugins/lookup/nfs_tmpfs_mountpoints.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from ansible.errors import AnsibleError +from ansible.plugins.loader import lookup_loader +from ansible.plugins.lookup import LookupBase + +from utils.storage.nfs import nested_tmpfs_mountpoints, volume_dir + + +class LookupModule(LookupBase): + """Tmpfs mountpoints in NFS volumes: {{ lookup('nfs_tmpfs_mountpoints', ...) }}.""" + + def run( + self, + terms: list[Any] | None, + variables: dict[str, Any] | None = None, + **kwargs: Any, + ) -> list[list[dict[str, str]]]: + """Return one list of ``{"path", "mode"}`` items, ``mode`` as an octal string. + + Args: + terms: none. + variables: the templating context; ``application_id`` is read from it. + kwargs: ``volumes``, the rendered top-level compose ``volumes:`` mapping. + """ + if terms: + raise AnsibleError("nfs_tmpfs_mountpoints lookup expects no terms") + if "volumes" not in kwargs: + raise AnsibleError("nfs_tmpfs_mountpoints lookup requires 'volumes'") + volumes = kwargs["volumes"] + if not isinstance(volumes, Mapping): + raise AnsibleError( + "nfs_tmpfs_mountpoints: 'volumes' must be the compose volumes " + f"mapping, got {type(volumes).__name__}" + ) + + vars_ = variables or self._templar.available_variables + application_id = str(self._templar.template(vars_["application_id"])).strip() + applications = lookup_loader.get( + "applications", loader=self._loader, templar=self._templar + ).run([], variables=vars_)[0] + if application_id not in applications: + raise AnsibleError( + f"nfs_tmpfs_mountpoints: unknown application_id '{application_id}'" + ) + + nfs_dirs = { + name: volume_dir(name, spec) + for name, spec in volumes.items() + if isinstance(spec, Mapping) and "driver_opts" in spec + } + try: + mountpoints = nested_tmpfs_mountpoints( + applications[application_id].get("volumes"), nfs_dirs + ) + except (TypeError, ValueError) as exc: + raise AnsibleError( + f"nfs_tmpfs_mountpoints: {application_id}: {exc}" + ) from exc + return [ + [ + {"path": mountpoint["path"], "mode": f"0{mountpoint['mode']:o}"} + for mountpoint in mountpoints + ] + ] diff --git a/roles/sys-svc-compose/tasks/utils/nfs_prep.yml b/roles/sys-svc-compose/tasks/utils/nfs_prep.yml index f250eb867e..fbbe15e71b 100644 --- a/roles/sys-svc-compose/tasks/utils/nfs_prep.yml +++ b/roles/sys-svc-compose/tasks/utils/nfs_prep.yml @@ -38,3 +38,14 @@ - item.0.value is mapping - item.0.value.driver_opts is defined - application_id not in lookup('roles_with_placement', 'manager') + + - name: "📁 Pre-create tmpfs mountpoints inside the NFS volumes of '{{ application_id }}'" + ansible.builtin.file: + path: "{{ STORAGE_NFS_STATE_PATH }}/{{ item.path }}" + state: directory + mode: "{{ item.mode }}" + delegate_to: "{{ groups['svc-storage-nfs-server'][0] }}" + run_once: true + loop: "{{ lookup('nfs_tmpfs_mountpoints', volumes=(lookup('compose_volumes') | from_yaml).volumes) }}" + loop_control: + label: "{{ item.path }} {{ item.mode }}" diff --git a/roles/web-app-weblate/meta/volumes.yml b/roles/web-app-weblate/meta/volumes.yml index 783d3823dd..cf015e84b6 100644 --- a/roles/web-app-weblate/meta/volumes.yml +++ b/roles/web-app-weblate/meta/volumes.yml @@ -12,6 +12,10 @@ data: cache: type: volume name: weblate_cache + nfs: + uid: "1000" + gid: "0" + mode: "0770" mounts: - service: weblate target: &app_cache_target /app/cache diff --git a/tests/unit/python/plugins/lookup/test_nfs_lookups.py b/tests/unit/python/plugins/lookup/test_nfs_lookups.py index ad70d95efa..c1ff3ade17 100644 --- a/tests/unit/python/plugins/lookup/test_nfs_lookups.py +++ b/tests/unit/python/plugins/lookup/test_nfs_lookups.py @@ -105,5 +105,52 @@ class TestNfsConfigBackedLookups(unittest.TestCase): self.assertEqual(result, [expected]) +class TestNfsTmpfsMountpointsLookup(unittest.TestCase): + def test_only_nfs_backed_volumes_yield_octal_string_modes(self): + vars_ = {"application_id": "web-app-demo"} + applications = { + "web-app-demo": { + "volumes": { + "cache": { + "type": "volume", + "name": "demo_cache", + "mounts": [{"service": "app", "target": "/app/cache"}], + }, + "local": { + "type": "volume", + "mounts": [{"service": "app", "target": "/app/local"}], + }, + "static": { + "type": "tmpfs", + "mounts": [ + { + "service": "app", + "target": "/app/cache/static", + "mode": "1777", + } + ], + }, + "scratch": { + "type": "tmpfs", + "mounts": [{"service": "app", "target": "/app/local/tmp"}], + }, + } + } + } + compose_volumes = { + "cache": { + "name": "demo_cache", + "driver": "local", + "driver_opts": {"type": "none", "o": "bind"}, + }, + "local": {}, + } + m, lm = _load("nfs_tmpfs_mountpoints", vars_) + with mock.patch.object(m, "lookup_loader") as loader: + loader.get.return_value = mock.MagicMock(run=lambda *a, **k: [applications]) + result = lm.run([], variables=vars_, volumes=compose_volumes) + self.assertEqual(result, [[{"path": "demo_cache/static", "mode": "01777"}]]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/python/roles/web-app-weblate/test_cache_volume_nfs_owner.py b/tests/unit/python/roles/web-app-weblate/test_cache_volume_nfs_owner.py new file mode 100644 index 0000000000..c0bfae76d2 --- /dev/null +++ b/tests/unit/python/roles/web-app-weblate/test_cache_volume_nfs_owner.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import unittest + +from utils.cache.files import PROJECT_ROOT, read_text +from utils.cache.yaml import load_yaml_str +from utils.roles.mapping import ROLE_FILE_META_VOLUMES + +VOLUMES = PROJECT_ROOT / "roles" / "web-app-weblate" / ROLE_FILE_META_VOLUMES + + +class TestCacheVolumeNfsOwner(unittest.TestCase): + def test_cache_volume_root_belongs_to_the_weblate_user(self) -> None: + volumes = load_yaml_str(read_text(str(VOLUMES))) + self.assertEqual( + volumes["cache"].get("nfs"), + {"uid": "1000", "gid": "0", "mode": "0770"}, + "the swarm NFS prep pre-creates the static tmpfs mountpoint inside " + "weblate_cache, so docker no longer copies the image's 1000:0 0770 " + "/app/cache onto the volume; the NFS prep has to set that owner and " + "mode itself, or uid 1000 cannot create /app/cache/avatar, ssh and " + "matplotlib", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/storage/test_nfs.py b/tests/unit/python/utils/storage/test_nfs.py index b0d0bdfba4..2c31e0edbe 100644 --- a/tests/unit/python/utils/storage/test_nfs.py +++ b/tests/unit/python/utils/storage/test_nfs.py @@ -1,15 +1,42 @@ import unittest +from utils.cache.files import PROJECT_ROOT, read_text +from utils.cache.yaml import load_yaml_str +from utils.roles.mapping import ROLE_FILE_META_VOLUMES from utils.storage.nfs import ( client_src, fstype, mount_opts, + nested_tmpfs_mountpoints, state_path, swarm_nfs_backed, volume_dir, ) +def _role_volumes(application_id): + """Return a role's ``meta/volumes.yml`` mapping. + + Args: + application_id: the role whose file is read. + """ + path = PROJECT_ROOT / "roles" / application_id / ROLE_FILE_META_VOLUMES + return load_yaml_str(read_text(str(path))) + + +def _every_volume_on_nfs(volumes_meta): + """Return the NFS directory of every volume that does not opt out of NFS. + + Args: + volumes_meta: a ``meta/volumes.yml`` mapping. + """ + return { + name: volume_dir(name, entry) + for name, entry in volumes_meta.items() + if entry.get("type", "volume") == "volume" and entry.get("nfs") is not False + } + + class TestNfsHelpers(unittest.TestCase): def test_state_path(self): self.assertEqual( @@ -155,5 +182,86 @@ class TestSwarmNfsBacked(unittest.TestCase): ) +class TestNestedTmpfsMountpoints(unittest.TestCase): + def test_weblate_static_mountpoint_carries_the_tmpfs_mode(self): + volumes = _role_volumes("web-app-weblate") + self.assertEqual( + nested_tmpfs_mountpoints(volumes, _every_volume_on_nfs(volumes)), + [{"path": "weblate_cache/static", "mode": 0o1777}], + "the static tmpfs of weblate, beat and worker sits inside the " + "weblate_cache volume; its mountpoint directory must exist with mode " + "1777 before the stack deploys, or uid 1000 cannot collectstatic", + ) + + def test_matrix_mautrix_workdir_is_not_nested_in_its_own_services_volume(self): + volumes = _role_volumes("web-app-matrix") + self.assertEqual( + nested_tmpfs_mountpoints(volumes, _every_volume_on_nfs(volumes)), + [], + "the mautrix tmpfs at /data shares its target with synapse_data, but " + "only synapse mounts that volume", + ) + + def test_a_sibling_path_is_not_nested(self): + volumes = { + "cache": { + "type": "volume", + "mounts": [{"service": "app", "target": "/app/cache"}], + }, + "scratch": { + "type": "tmpfs", + "mounts": [{"service": "app", "target": "/app/cachex"}], + }, + } + self.assertEqual(nested_tmpfs_mountpoints(volumes, {"cache": "app_cache"}), []) + + def test_the_innermost_volume_holds_the_mountpoint(self): + volumes = { + "app": { + "type": "volume", + "mounts": [{"service": "app", "target": "/app"}], + }, + "cache": { + "type": "volume", + "mounts": [{"service": "app", "target": "/app/cache/"}], + }, + "static": { + "type": "tmpfs", + "mounts": [ + {"service": "app", "target": "/app/cache/static", "mode": "0700"} + ], + }, + } + nfs_dirs = {"app": "app_app", "cache": "app_cache"} + self.assertEqual( + nested_tmpfs_mountpoints(volumes, nfs_dirs), + [{"path": "app_cache/static", "mode": 0o700}], + ) + + def test_a_nested_tmpfs_without_mode_fails_loudly(self): + volumes = { + "cache": { + "type": "volume", + "mounts": [{"service": "app", "target": "/app/cache"}], + }, + "static": { + "type": "tmpfs", + "mounts": [{"service": "app", "target": "/app/cache/static"}], + }, + } + with self.assertRaisesRegex(ValueError, "declares no 'mode'"): + nested_tmpfs_mountpoints(volumes, {"cache": "app_cache"}) + + def test_every_role_declares_a_mode_on_its_nested_tmpfs_mounts(self): + for path in sorted( + (PROJECT_ROOT / "roles").glob(f"*/{ROLE_FILE_META_VOLUMES}") + ): + volumes = load_yaml_str(read_text(str(path))) + if not isinstance(volumes, dict): + continue + with self.subTest(role=path.parent.parent.name): + nested_tmpfs_mountpoints(volumes, _every_volume_on_nfs(volumes)) + + if __name__ == "__main__": unittest.main() diff --git a/utils/storage/nfs.py b/utils/storage/nfs.py index 80a0683eca..e33c487504 100644 --- a/utils/storage/nfs.py +++ b/utils/storage/nfs.py @@ -6,6 +6,7 @@ from pathlib import PurePosixPath from utils import PROJECT_ROOT from utils.cache.files import read_text +from utils.roles.applications.mounts import normalize_volumes_meta from utils.roles.mapping import ROLE_FILE_META_SERVICES STATE_SUBDIR = "infinito-state" @@ -138,6 +139,71 @@ def volume_dir(semantic_name, entry): return str(pinned) if pinned else str(semantic_name) +def _tmpfs_mode(value): + if isinstance(value, bool) or not isinstance(value, (int, str)): + raise TypeError(f"tmpfs mode must be an octal string or int, got {value!r}") + return value if isinstance(value, int) else int(value, 8) + + +def nested_tmpfs_mountpoints(volumes_meta, nfs_dirs): + """Tmpfs mountpoints whose innermost volume of the same service is NFS-backed. + + Args: + volumes_meta: the role's ``meta/volumes.yml`` mapping. + nfs_dirs: NFS state directory per semantic name of every NFS-backed volume. + + Returns: + One ``{"path": str, "mode": int}`` per mountpoint, sorted by ``path``, + which is relative to the NFS state path. + + Raises: + ValueError: a nested tmpfs declares no ``mode``, or two tmpfs mounts + declare different modes for the same mountpoint. + TypeError: a nested tmpfs ``mode`` is neither an octal string nor an int. + """ + entries = normalize_volumes_meta(volumes_meta) + parents: dict[str, list[tuple[str, str]]] = {} + for name, entry in entries.items(): + if entry["type"] != "volume": + continue + for mount in entry.get("mounts") or []: + parents.setdefault(mount["service"], []).append( + (str(mount["target"]), name) + ) + + modes: dict[str, int] = {} + for name, entry in entries.items(): + if entry["type"] != "tmpfs": + continue + for mount in entry.get("mounts") or []: + target = str(mount["target"]) + enclosing = [ + (parent.rstrip("/"), volume) + for parent, volume in parents.get(mount["service"], []) + if target == parent or target.startswith(parent.rstrip("/") + "/") + ] + if not enclosing: + continue + base, volume = max(enclosing, key=lambda pair: len(pair[0])) + if volume not in nfs_dirs: + continue + if "mode" not in mount: + raise ValueError( + f"tmpfs {name!r} of service {mount['service']!r} at {target} " + f"lies inside the NFS-backed volume {volume!r} and declares " + f"no 'mode'" + ) + relative = target[len(base) :].lstrip("/") + path = str(PurePosixPath(nfs_dirs[volume], relative)) + mode = _tmpfs_mode(mount["mode"]) + if modes.setdefault(path, mode) != mode: + raise ValueError( + f"tmpfs mountpoint {path} is declared with modes " + f"{modes[path]:o} and {mode:o}" + ) + return [{"path": path, "mode": mode} for path, mode in sorted(modes.items())] + + def backup_disabled_dirs(applications, *, deployment_mode, storage_backend): """State-path directories of the NFS-backed volumes declared ``backup: false``.