diff --git a/library/copy_batch.py b/library/copy_batch.py
new file mode 100644
index 0000000000..acd0af4fb0
--- /dev/null
+++ b/library/copy_batch.py
@@ -0,0 +1,332 @@
+"""Write the differing entries of a directory copy in one module call.
+
+Every path is handled like ``ansible.builtin.copy`` handles one entry of a
+recursive copy: a missing parent directory is created and receives
+``directory_mode`` while an existing one stays as found, a file is replaced
+through ``atomic_move`` and receives ``mode``, and a leaf directory is ensured
+like ``ansible.builtin.file`` with ``state: directory``. Files are handled in
+the order of ``tree``, which is the order the copy action walks the source
+in, and leaf directories after them.
+Nothing is written when a file path exists as anything but a regular file, a
+leaf directory path as anything but a directory, a missing path lies below
+anything but a directory, or two paths of ``tree`` name the same remote
+entry; the call then only reports ``refused``.
+
+Args:
+ src: tar archive holding exactly one regular member per entry of ``files``.
+ dest: absolute destination directory.
+ files: list of ``{path, checksum}`` in source walk order, ``path``
+ relative to ``dest`` and ``checksum`` the SHA-1 of its archive member.
+ settled: relative paths of regular files whose content is in place; only
+ ``mode`` is enforced on them.
+ directories: relative paths of leaf directories to ensure.
+ tree: relative path of every file and leaf directory of the source tree,
+ in walk order; holds every path of ``files``, ``settled`` and
+ ``directories``.
+ mode: mode of every path in ``files`` and ``settled``.
+ directory_mode: mode of every directory this call creates and of every
+ path in ``directories``; ``None`` keeps the mode the umask gives.
+ unsafe_writes: the file common argument of the same name, handed to
+ ``atomic_move`` as the copy module does.
+
+Returns:
+ changed: whether any file, mode or directory changed.
+ refused: the reasons nothing was written; present only then.
+"""
+
+import hashlib
+import os
+import shutil
+import stat
+import tarfile
+import tempfile
+from pathlib import Path, PurePosixPath
+
+from ansible.module_utils.basic import FILE_COMMON_ARGUMENTS, AnsibleModule
+
+ARGUMENT_SPEC = {
+ "src": {"type": "path", "required": True},
+ "dest": {"type": "path", "required": True},
+ "files": {
+ "type": "list",
+ "elements": "dict",
+ "required": True,
+ "options": {
+ "path": {"type": "str", "required": True},
+ "checksum": {"type": "str", "required": True},
+ },
+ },
+ "settled": {"type": "list", "elements": "str", "required": True},
+ "directories": {"type": "list", "elements": "str", "required": True},
+ "tree": {"type": "list", "elements": "str", "required": True},
+ "mode": {"type": "raw", "required": True},
+ "directory_mode": {"type": "raw"},
+ "unsafe_writes": FILE_COMMON_ARGUMENTS["unsafe_writes"],
+}
+
+_CHUNK = 1 << 20
+
+
+def is_relative(name):
+ """Tell whether *name* is a normalised path strictly below the destination.
+
+ Args:
+ name: relative path from the module arguments or the archive.
+ """
+ path = PurePosixPath(name)
+ return (
+ bool(path.parts)
+ and str(path) == name
+ and not path.is_absolute()
+ and ".." not in path.parts
+ )
+
+
+def blocker(path, is_kind, kind, may_be_missing):
+ """Name what keeps *path* from taking an entry of the wanted type.
+
+ Args:
+ path: absolute destination path.
+ is_kind: ``stat.S_ISREG`` or ``stat.S_ISDIR``; an existing entry must
+ pass it, so a symlink never does.
+ kind: the wanted type in words.
+ may_be_missing: whether a missing *path* below a directory is fine.
+
+ Returns:
+ ``None`` when *path* can take the entry, else the reason it cannot.
+ """
+ try:
+ info = path.lstat()
+ except FileNotFoundError:
+ if not may_be_missing:
+ return f"{path} is missing"
+ parent = path.parent
+ while not os.path.lexists(parent):
+ parent = parent.parent
+ return None if parent.is_dir() else f"{parent} is not a directory"
+ except OSError as error:
+ return f"{path}: {error.strerror}"
+ return None if is_kind(info.st_mode) else f"{path} is not {kind}"
+
+
+def aliases(dest, tree):
+ """Name the paths of *tree* that reach the same remote entry.
+
+ Args:
+ dest: the destination directory.
+ tree: relative paths of every file and leaf directory of the source.
+
+ Returns:
+ One reason per path that reaches an entry an earlier path reaches.
+ """
+ seen = {}
+ reasons = []
+ for name in tree:
+ path = dest / name
+ try:
+ info = path.stat()
+ key = (info.st_dev, info.st_ino)
+ except OSError:
+ key = str(Path(os.path.realpath(path.parent)) / path.name)
+ if key in seen:
+ reasons.append(f"{dest / seen[key]} and {path} reach the same entry")
+ else:
+ seen[key] = name
+ return reasons
+
+
+def refusals(dest, files, settled, directories, tree):
+ """Collect what keeps any path of the call from taking its entry.
+
+ Args:
+ dest: the destination directory.
+ files: relative paths of the files to write.
+ settled: relative paths of the files to keep.
+ directories: relative paths of the leaf directories to ensure.
+ tree: relative paths of every file and leaf directory of the source.
+
+ Returns:
+ The reasons, empty when every path can take its entry.
+ """
+ checks = [
+ *((name, stat.S_ISREG, "a regular file", True) for name in files),
+ *((name, stat.S_ISREG, "a regular file", False) for name in settled),
+ *((name, stat.S_ISDIR, "a directory", True) for name in directories),
+ ]
+ reasons = [
+ blocker(dest / name, is_kind, kind, may_be_missing)
+ for name, is_kind, kind, may_be_missing in checks
+ ]
+ return [reason for reason in reasons if reason] + aliases(dest, tree)
+
+
+def members(module, archive, expected):
+ """Map every path of *expected* to its archive member, failing on any other.
+
+ Args:
+ module: the running module.
+ archive: the open tar archive.
+ expected: mapping of relative path to the SHA-1 of its member.
+
+ Returns:
+ Mapping of relative path to its tar member.
+ """
+ found = {}
+ for member in archive.getmembers():
+ name = member.name
+ if not is_relative(name):
+ module.fail_json(msg=f"archive member {name!r} leaves the destination")
+ if name not in expected or name in found:
+ module.fail_json(msg=f"archive member {name!r} is unexpected or repeated")
+ if not member.isreg():
+ module.fail_json(msg=f"archive member {name!r} is not a regular file")
+ found[name] = member
+ missing = sorted(set(expected) - set(found))
+ if missing:
+ module.fail_json(msg=f"archive lacks {', '.join(missing)}")
+ return found
+
+
+def sha1(archive, member):
+ """Return the SHA-1 of the content of *member*.
+
+ Args:
+ archive: the open tar archive.
+ member: a regular tar member.
+ """
+ digest = hashlib.sha1(usedforsecurity=False)
+ with archive.extractfile(member) as handle:
+ while chunk := handle.read(_CHUNK):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def attributes(module, path, mode, changed):
+ """Apply *mode* to *path* like the copy and file modules do.
+
+ Args:
+ module: the running module.
+ path: the file or directory.
+ mode: the wanted mode, or ``None`` to leave it.
+ changed: the change state so far.
+
+ Returns:
+ The change state including this call.
+ """
+ file_args = module.load_file_common_arguments({"path": str(path), "mode": mode})
+ return module.set_fs_attributes_if_different(file_args, changed)
+
+
+def make_directories(module, directory, directory_mode):
+ """Create *directory* and its missing parents, each with *directory_mode*.
+
+ Args:
+ module: the running module.
+ directory: the directory that must exist.
+ directory_mode: mode of every directory created, or ``None``.
+
+ Returns:
+ Whether a directory was missing.
+ """
+ missing = []
+ while not os.path.lexists(directory):
+ missing.append(directory)
+ directory = directory.parent
+ for path in reversed(missing):
+ try:
+ path.mkdir()
+ except FileExistsError:
+ if not path.is_dir():
+ raise
+ attributes(module, path, directory_mode, True)
+ return bool(missing)
+
+
+def write(module, archive, member, target, workdir):
+ """Replace *target* atomically with the content of *member*.
+
+ Args:
+ module: the running module, whose ``atomic_move`` keeps the owner,
+ group and SELinux context of a replaced file.
+ archive: the open tar archive.
+ member: the tar member holding the new content.
+ target: destination path.
+ workdir: directory the content is staged in before the move.
+ """
+ descriptor, staged = tempfile.mkstemp(dir=workdir)
+ try:
+ with os.fdopen(descriptor, "wb") as out, archive.extractfile(member) as data:
+ shutil.copyfileobj(data, out)
+ module.atomic_move(
+ staged, str(target), unsafe_writes=module.params["unsafe_writes"]
+ )
+ finally:
+ Path(staged).unlink(missing_ok=True)
+
+
+def main():
+ module = AnsibleModule(argument_spec=ARGUMENT_SPEC)
+ params = module.params
+ dest = Path(params["dest"])
+ files = [entry["path"] for entry in params["files"]]
+ expected = {entry["path"]: entry["checksum"] for entry in params["files"]}
+ settled, directories = params["settled"], params["directories"]
+ kept = set(settled)
+ directory_mode = params["directory_mode"]
+ names = [*files, *settled, *directories, *params["tree"]]
+ unsafe = sorted({name for name in names if not is_relative(name)})
+ if unsafe:
+ module.fail_json(msg=f"paths leave {dest}: {', '.join(unsafe)}")
+ if len(expected) != len(files):
+ module.fail_json(msg="files repeat a path")
+ outside = sorted(set(names) - set(params["tree"]))
+ if outside:
+ module.fail_json(msg=f"paths missing from tree: {', '.join(outside)}")
+ refused = refusals(dest, files, settled, directories, params["tree"])
+ if refused:
+ module.exit_json(changed=False, refused=refused)
+
+ changed = False
+ with tarfile.open(params["src"], mode="r:") as archive:
+ found = members(module, archive, expected)
+ mismatched = [
+ name for name in files if sha1(archive, found[name]) != expected[name]
+ ]
+ if mismatched:
+ module.fail_json(
+ msg=f"archived files do not match their checksum: {', '.join(mismatched)}"
+ )
+ for name in params["tree"]:
+ target = dest / name
+ if name in kept:
+ changed = attributes(module, target, params["mode"], changed)
+ if name not in expected:
+ continue
+ try:
+ changed = (
+ make_directories(module, target.parent, directory_mode) or changed
+ )
+ write(module, archive, found[name], target, module.tmpdir)
+ except Exception as error: # noqa: BLE001 - atomic_move raises a bare Exception
+ module.fail_json(
+ msg=f"Failed to write {target}: {error}",
+ path=str(target),
+ changed=changed,
+ )
+ changed = attributes(module, target, params["mode"], True)
+ for name in directories:
+ path = dest / name
+ try:
+ created = make_directories(module, path, directory_mode)
+ except OSError as error:
+ module.fail_json(
+ msg=f"Failed to create {path}: {error}",
+ path=str(path),
+ changed=changed,
+ )
+ changed = created or attributes(module, path, directory_mode, changed)
+ module.exit_json(changed=changed)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/plugins/action/stack_host_copy.py b/plugins/action/stack_host_copy.py
index 183bb9adbe..1102a123bd 100755
--- a/plugins/action/stack_host_copy.py
+++ b/plugins/action/stack_host_copy.py
@@ -1,11 +1,16 @@
#!/usr/bin/env python3
from __future__ import annotations
+import hashlib
+import io
import os
import re
+import tarfile
+import tempfile
from pathlib import Path, PurePath
from typing import Any
+from ansible import constants
from ansible.errors import AnsibleError
from ansible.module_utils.parsing.convert_bool import boolean as _to_bool
from ansible.plugins.action import copy as _copy_action
@@ -29,6 +34,8 @@ _TREE_CHECK_ARGS = frozenset(
}
)
_OCTAL_MODE = re.compile(r"^[0-7]{3,4}$")
+_ARCHIVE = ".source.tar"
+_ARCHIVE_LIMIT = 16 * 1024 * 1024
def _octal(mode: Any) -> int | None:
@@ -83,9 +90,9 @@ class ActionModule(CopyActionModule):
}
self._task.args.setdefault("mode", "0644")
- unchanged = self._unchanged_tree(task_vars)
- if unchanged is not None:
- return unchanged
+ settled = self._settled_tree(task_vars)
+ if settled is not None:
+ return settled
walk = _copy_action._walk_dirs
_copy_action._walk_dirs = _walk_dirs_without_bytecode_caches
try:
@@ -93,19 +100,22 @@ class ActionModule(CopyActionModule):
finally:
_copy_action._walk_dirs = walk
- def _unchanged_tree(self, task_vars: dict[str, Any]) -> dict[str, Any] | None:
- """Settle a directory copy whose entries all match in one remote call.
+ def _settled_tree(self, task_vars: dict[str, Any]) -> dict[str, Any] | None:
+ """Settle a directory copy in one checksum call and one write call.
- The copy action spends a stat and a file module call per file, and a
- file module call per directory holding no file, even when nothing
- differs; any difference, and any argument or source shape not
- modelled here, leaves the decision to the regular copy.
+ The copy action spends a stat and a file module call per file, plus a
+ transfer and a copy module call per differing file, and a file module
+ call per directory holding no file. Here one remote call compares the
+ whole tree; when entries differ outside check and diff mode, the
+ differing files travel in one archive and one module call writes them.
+ Any argument or source shape not modelled here leaves the decision to
+ the regular copy.
Args:
- task_vars: Variables of the task, handed to the remote call.
+ task_vars: Variables of the task, handed to the remote calls.
Returns:
- The unchanged copy result, or ``None`` when the regular copy runs.
+ The copy result, or ``None`` when the regular copy runs.
"""
args = self._task.args
src, dest, mode = args.get("src"), args.get("dest"), _octal(args.get("mode"))
@@ -116,6 +126,7 @@ class ActionModule(CopyActionModule):
or not src.endswith(os.path.sep)
or not isinstance(dest, str)
or not dest.startswith("/")
+ or "$" in dest
or mode is None
or not _to_bool(args.get("force", True), strict=False)
):
@@ -146,7 +157,7 @@ class ActionModule(CopyActionModule):
return None
dest = shell.join_path(dest, "")
- directories = [shell.join_path(dest, rel) for rel in leaves]
+ directories = {shell.join_path(dest, rel): rel for rel in leaves}
decrypt = _to_bool(args.get("decrypt", True), strict=False)
wanted = {}
for full, rel in tree["files"]:
@@ -155,7 +166,7 @@ class ActionModule(CopyActionModule):
except AnsibleError:
return None
try:
- wanted[shell.join_path(dest, rel)] = checksum(real)
+ wanted[shell.join_path(dest, rel)] = (full, rel, checksum(real))
finally:
self._loader.cleanup_tmp_file(real)
@@ -167,21 +178,148 @@ class ActionModule(CopyActionModule):
states = found.get("states")
if found.get("failed") or not isinstance(states, dict):
return None
- for path, digest in wanted.items():
+ content, settled = {}, []
+ for path, (full, rel, digest) in wanted.items():
state = states.get(path) or {}
- if (
- not state.get("isreg")
- or state.get("checksum") != digest
- or _octal(state.get("mode")) != mode
- ):
- return None
- for path in directories:
+ if not state.get("isreg") or state.get("checksum") != digest:
+ content[rel] = full
+ elif _octal(state.get("mode")) != mode:
+ settled.append(rel)
+ drifted = []
+ for path, rel in directories.items():
state = states.get(path) or {}
if not state.get("isdir") or (
directory_mode is not None
and _octal(state.get("mode")) != _octal(directory_mode)
):
- return None
+ drifted.append(rel)
+ if not (content or settled or drifted):
+ self._remove_tmp_path(shell.tmpdir)
+ return self._ensure_invocation(
+ {"changed": False, "dest": dest, "src": source}
+ )
+ if self._task.check_mode or self._task.diff or self._is_become_unprivileged():
+ return None
+ return self._write_tree(
+ task_vars,
+ source=source,
+ dest=dest,
+ content=content,
+ settled=settled,
+ directories=drifted,
+ tree=[*(rel for _, rel in tree["files"]), *leaves],
+ decrypt=decrypt,
+ )
+
+ def _write_tree(
+ self,
+ task_vars: dict[str, Any],
+ *,
+ source: str,
+ dest: str,
+ content: dict[str, str],
+ settled: list[str],
+ directories: list[str],
+ tree: list[str],
+ decrypt: bool,
+ ) -> dict[str, Any] | None:
+ """Write the differing entries of a directory copy in one module call.
+
+ Args:
+ task_vars: Variables of the task, handed to the remote call.
+ source: The local source directory, with a trailing slash.
+ dest: The remote destination directory, with a trailing slash.
+ content: Relative path to local path of every file whose remote
+ content differs or is missing, in walk order.
+ settled: Relative paths of the files whose mode alone differs.
+ directories: Relative paths of the leaf directories that are
+ missing or whose mode differs.
+ tree: Relative paths of every file and leaf directory of the
+ source, for the remote check that no two reach one entry.
+ decrypt: Whether vaulted source files are decrypted.
+
+ Returns:
+ The copy result, or ``None`` when the archive would exceed
+ ``_ARCHIVE_LIMIT`` or the module refuses a remote entry, and the
+ regular copy runs.
+ """
+ shell = self._connection._shell
+ descriptor, archive = tempfile.mkstemp(
+ dir=constants.DEFAULT_LOCAL_TMP, prefix="."
+ )
+ try:
+ with os.fdopen(descriptor, "wb") as handle:
+ files = self._pack(handle, content, decrypt)
+ if files is None:
+ return None
+ if shell.tmpdir is None:
+ self._make_tmp_path()
+ remote = self._transfer_file(
+ archive, shell.join_path(shell.tmpdir, _ARCHIVE)
+ )
+ finally:
+ Path(archive).unlink()
+ self._fixup_perms2((shell.tmpdir, remote))
+ written = self._execute_module(
+ module_name="ansible.legacy.copy_batch",
+ module_args={
+ "src": remote,
+ "dest": dest,
+ "files": files,
+ "settled": settled,
+ "directories": directories,
+ "tree": tree,
+ "mode": self._task.args["mode"],
+ "directory_mode": self._task.args.get("directory_mode"),
+ },
+ task_vars=task_vars,
+ )
+ if written.get("refused"):
+ return None
self._remove_tmp_path(shell.tmpdir)
- return self._ensure_invocation({"changed": False, "dest": dest, "src": source})
+ if written.get("failed"):
+ return self._ensure_invocation(written)
+ return self._ensure_invocation(
+ {"changed": written["changed"], "dest": dest, "src": source}
+ )
+
+ def _pack(
+ self, handle: Any, content: dict[str, str], decrypt: bool
+ ) -> list[dict[str, str]] | None:
+ """Write the files of *content* as regular members of a tar archive.
+
+ Args:
+ handle: Binary file object the archive is written to.
+ content: Relative path to local path of every file, in walk order.
+ decrypt: Whether vaulted source files are decrypted.
+
+ Returns:
+ ``{path, checksum}`` of every member in walk order, the checksum
+ taken from the bytes packed, or ``None`` once the members exceed
+ ``_ARCHIVE_LIMIT`` bytes.
+ """
+ files, size = [], 0
+ with tarfile.open(fileobj=handle, mode="w") as bundle:
+ for rel, full in content.items():
+ real = self._loader.get_real_file(full, decrypt=decrypt)
+ try:
+ with Path(real).open("rb") as source:
+ data = source.read(_ARCHIVE_LIMIT + 1 - size)
+ finally:
+ self._loader.cleanup_tmp_file(real)
+ size += len(data)
+ if size > _ARCHIVE_LIMIT:
+ return None
+ member = tarfile.TarInfo(rel)
+ member.size = len(data)
+ bundle.addfile(member, io.BytesIO(data))
+ files.append(
+ {
+ "path": rel,
+ "checksum": hashlib.sha1(
+ data, usedforsecurity=False
+ ).hexdigest(),
+ }
+ )
+ return files
diff --git a/tests/unit/python/library/test_copy_batch.py b/tests/unit/python/library/test_copy_batch.py
new file mode 100644
index 0000000000..0de9a8fed4
--- /dev/null
+++ b/tests/unit/python/library/test_copy_batch.py
@@ -0,0 +1,714 @@
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import importlib.util
+import io
+import json
+import os
+import stat
+import sys
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path
+from typing import TYPE_CHECKING, ClassVar
+from unittest.mock import patch
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.testing import patch_module_args
+from ansible.modules import copy as copy_module
+from ansible.modules import file as file_module
+
+from . import PROJECT_ROOT
+
+if TYPE_CHECKING:
+ from types import ModuleType
+
+sys.path.insert(0, str(PROJECT_ROOT))
+
+MODULE_PATH = PROJECT_ROOT / "library" / "copy_batch.py"
+
+
+def _load_copy_batch() -> ModuleType:
+ spec = importlib.util.spec_from_file_location(
+ "library_copy_batch", str(MODULE_PATH)
+ )
+ if spec is None or spec.loader is None:
+ raise ImportError(f"Could not load: {MODULE_PATH}")
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod) # type: ignore[attr-defined]
+ return mod
+
+
+_MOD = _load_copy_batch()
+
+
+def _sha1(data):
+ return hashlib.sha1(data, usedforsecurity=False).hexdigest()
+
+
+def _umask():
+ current = os.umask(0)
+ os.umask(current)
+ return current
+
+
+def _call(main, args):
+ buffer = io.StringIO()
+ with (
+ patch_module_args(args),
+ contextlib.redirect_stdout(buffer),
+ contextlib.suppress(SystemExit),
+ ):
+ main()
+ return json.loads(buffer.getvalue())
+
+
+class _Tree(unittest.TestCase):
+ def setUp(self):
+ self._tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self._tmp.name)
+ self.dest = self.root / "dest"
+ self.dest.mkdir()
+ self.work = self.root / "work"
+ self.work.mkdir()
+ self.archive = self.root / "archive.tar"
+
+ def tearDown(self):
+ self._tmp.cleanup()
+
+ def _pack(self, files, *, names=None, kinds=None):
+ with tarfile.open(self.archive, "w") as bundle:
+ for name, data in files.items():
+ member = tarfile.TarInfo((names or {}).get(name, name))
+ member.type = (kinds or {}).get(name, tarfile.REGTYPE)
+ member.size = len(data) if member.isreg() else 0
+ bundle.addfile(member, io.BytesIO(data) if member.isreg() else None)
+ return {name: _sha1(data) for name, data in files.items()}
+
+ def _run(
+ self,
+ files,
+ *,
+ settled=(),
+ directories=(),
+ mode="0644",
+ directory_mode=None,
+ digests=None,
+ names=None,
+ kinds=None,
+ tree=None,
+ ):
+ packed = self._pack(files, names=names, kinds=kinds)
+ listed = packed if digests is None else digests
+ return _call(
+ _MOD.main,
+ {
+ "src": str(self.archive),
+ "dest": f"{self.dest}/",
+ "files": [
+ {"path": name, "checksum": digest}
+ for name, digest in listed.items()
+ ],
+ "settled": list(settled),
+ "directories": list(directories),
+ "tree": [*listed, *settled, *directories] if tree is None else tree,
+ "mode": mode,
+ "directory_mode": directory_mode,
+ "_ansible_tmpdir": str(self.work),
+ },
+ )
+
+ def _write(self, rel, data, mode):
+ path = self.dest / rel
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(data)
+ path.chmod(mode)
+ return path
+
+ @staticmethod
+ def _mode(path):
+ return stat.S_IMODE(path.lstat().st_mode)
+
+ @staticmethod
+ def _bytes(path):
+ return path.read_bytes() # nocheck: cache-read
+
+
+class TestWrites(_Tree):
+ def test_replaces_and_creates_files_with_mode(self):
+ self._write("icon.png", b"old", 0o600)
+
+ result = self._run({"icon.png": b"new", "nested/icon.svg": b""})
+
+ self.assertTrue(result["changed"])
+ self.assertNotIn("refused", result)
+ self.assertEqual(self._bytes(self.dest / "icon.png"), b"new")
+ self.assertEqual(self._bytes(self.dest / "nested/icon.svg"), b"")
+ self.assertEqual(self._mode(self.dest / "icon.png"), 0o644)
+ self.assertEqual(self._mode(self.dest / "nested/icon.svg"), 0o644)
+
+ def test_created_parents_receive_the_directory_mode(self):
+ self._run({"a/b/icon.svg": b""}, directory_mode="0750")
+
+ self.assertEqual(self._mode(self.dest / "a"), 0o750)
+ self.assertEqual(self._mode(self.dest / "a/b"), 0o750)
+
+ def test_existing_parent_keeps_its_mode(self):
+ (self.dest / "a").mkdir()
+ (self.dest / "a").chmod(0o701)
+
+ self._run({"a/b/icon.svg": b""}, directory_mode="0750")
+
+ self.assertEqual(self._mode(self.dest / "a"), 0o701)
+ self.assertEqual(self._mode(self.dest / "a/b"), 0o750)
+
+ def test_unset_directory_mode_keeps_the_umask_mode(self):
+ self._run({"a/icon.svg": b""}, directory_mode=None)
+
+ self.assertEqual(self._mode(self.dest / "a"), 0o777 & ~_umask())
+
+ def test_existing_leaf_directory_receives_the_directory_mode(self):
+ (self.dest / "leaf").mkdir()
+ (self.dest / "leaf").chmod(0o700)
+
+ result = self._run({}, directories=["leaf"], directory_mode="0750")
+
+ self.assertTrue(result["changed"])
+ self.assertEqual(self._mode(self.dest / "leaf"), 0o750)
+
+ def test_missing_leaf_directory_is_created_with_its_parents(self):
+ result = self._run({}, directories=["a/leaf"], directory_mode="0750")
+
+ self.assertTrue(result["changed"])
+ self.assertEqual(self._mode(self.dest / "a"), 0o750)
+ self.assertEqual(self._mode(self.dest / "a/leaf"), 0o750)
+
+ def test_settled_file_keeps_its_content_and_inode(self):
+ path = self._write("icon.png", b"png", 0o600)
+ inode = path.stat().st_ino
+
+ result = self._run({}, settled=["icon.png"])
+
+ self.assertTrue(result["changed"])
+ self.assertEqual(path.stat().st_ino, inode)
+ self.assertEqual(self._bytes(path), b"png")
+ self.assertEqual(self._mode(path), 0o644)
+
+ def test_settled_file_with_its_mode_reports_no_change(self):
+ self._write("icon.png", b"png", 0o644)
+
+ result = self._run({}, settled=["icon.png"])
+
+ self.assertFalse(result["changed"])
+
+ def test_replaced_file_gets_the_owner_of_the_file_it_replaces(self):
+ path = self._write("icon.png", b"old", 0o600)
+ owner = (path.stat().st_uid, path.stat().st_gid)
+
+ with (
+ patch.object(
+ AnsibleModule,
+ "atomic_move",
+ autospec=True,
+ side_effect=AnsibleModule.atomic_move,
+ ) as move,
+ patch("os.chown", wraps=os.chown) as chown,
+ ):
+ self._run({"icon.png": b"new"})
+
+ self.assertEqual(move.call_args.args[2], str(path))
+ self.assertNotIn("keep_dest_attrs", move.call_args.kwargs)
+ self.assertIn(owner, [call.args[1:] for call in chown.call_args_list])
+ self.assertEqual((path.stat().st_uid, path.stat().st_gid), owner)
+
+ def test_files_outside_the_call_stay(self):
+ other = self._write("other.png", b"other", 0o600)
+
+ self._run({"icon.png": b"new"})
+
+ self.assertEqual(self._bytes(other), b"other")
+ self.assertEqual(self._mode(other), 0o600)
+
+ def test_symlinked_parent_directory_is_written_through(self):
+ real = self.root / "real"
+ real.mkdir()
+ (self.dest / "linked").symlink_to(real)
+
+ result = self._run({"linked/icon.svg": b""})
+
+ self.assertTrue(result["changed"])
+ self.assertEqual(self._bytes(real / "icon.svg"), b"")
+
+ def test_staged_copies_leave_no_trace(self):
+ self._run({"icon.png": b"new", "nested/icon.svg": b""})
+
+ self.assertEqual(list(self.work.iterdir()), [])
+ self.assertEqual(
+ sorted(p.name for p in self.dest.iterdir()), ["icon.png", "nested"]
+ )
+
+
+class TestRefusals(_Tree):
+ def _assert_refused(self, result, path):
+ self.assertFalse(result["changed"])
+ self.assertTrue(any(str(path) in reason for reason in result["refused"]))
+ self.assertFalse((self.dest / "fresh.png").exists())
+
+ def test_symlink_in_place_of_a_file_is_refused_and_kept(self):
+ target = self.root / "target.png"
+ target.write_bytes(b"target")
+ (self.dest / "icon.png").symlink_to(target)
+
+ result = self._run({"icon.png": b"new", "fresh.png": b"fresh"})
+
+ self._assert_refused(result, self.dest / "icon.png")
+ self.assertTrue((self.dest / "icon.png").is_symlink())
+ self.assertEqual(self._bytes(target), b"target")
+
+ def test_dangling_symlink_in_place_of_a_file_is_refused(self):
+ (self.dest / "icon.png").symlink_to(self.root / "absent")
+
+ result = self._run({"icon.png": b"new", "fresh.png": b"fresh"})
+
+ self._assert_refused(result, self.dest / "icon.png")
+
+ def test_directory_in_place_of_a_file_is_refused(self):
+ (self.dest / "icon.png").mkdir()
+
+ result = self._run({"icon.png": b"new", "fresh.png": b"fresh"})
+
+ self._assert_refused(result, self.dest / "icon.png")
+ self.assertTrue((self.dest / "icon.png").is_dir())
+
+ def test_fifo_in_place_of_a_file_is_refused(self):
+ os.mkfifo(self.dest / "icon.png")
+
+ result = self._run({"icon.png": b"new", "fresh.png": b"fresh"})
+
+ self._assert_refused(result, self.dest / "icon.png")
+
+ def test_file_in_place_of_a_parent_directory_is_refused(self):
+ self._write("nested", b"file", 0o644)
+
+ result = self._run({"nested/icon.svg": b"", "fresh.png": b"fresh"})
+
+ self._assert_refused(result, self.dest / "nested")
+ self.assertEqual(self._bytes(self.dest / "nested"), b"file")
+
+ def test_dangling_symlink_in_place_of_a_parent_directory_is_refused(self):
+ (self.dest / "nested").symlink_to(self.root / "absent")
+
+ result = self._run({"nested/a/icon.svg": b"", "fresh.png": b"fresh"})
+
+ self._assert_refused(result, self.dest / "nested")
+ self.assertFalse((self.root / "absent").exists())
+
+ def test_symlink_in_place_of_a_leaf_directory_is_refused(self):
+ (self.root / "real").mkdir()
+ (self.dest / "leaf").symlink_to(self.root / "real")
+
+ result = self._run(
+ {"fresh.png": b"fresh"}, directories=["leaf"], directory_mode="0750"
+ )
+
+ self._assert_refused(result, self.dest / "leaf")
+ self.assertNotEqual(self._mode(self.root / "real"), 0o750)
+
+ def test_file_in_place_of_a_leaf_directory_is_refused(self):
+ self._write("leaf", b"", 0o644)
+
+ result = self._run({"fresh.png": b"fresh"}, directories=["leaf"])
+
+ self._assert_refused(result, self.dest / "leaf")
+
+ def test_symlink_loop_in_place_of_a_parent_directory_is_refused(self):
+ (self.dest / "loop").symlink_to(self.dest / "loop")
+
+ result = self._run({"loop/icon.svg": b"", "fresh.png": b"fresh"})
+
+ self._assert_refused(result, self.dest / "loop")
+
+ def test_missing_settled_file_is_refused(self):
+ result = self._run({"fresh.png": b"fresh"}, settled=["icon.png"])
+
+ self._assert_refused(result, self.dest / "icon.png")
+
+
+class TestFailures(_Tree):
+ def _assert_failed(self, result, needle):
+ self.assertTrue(result["failed"])
+ self.assertIn(needle, result["msg"])
+ self.assertFalse((self.dest / "icon.png").exists())
+
+ def test_checksum_mismatch_fails_naming_the_file_and_writes_nothing(self):
+ digests = {"icon.png": _sha1(b"new"), "nested/icon.svg": _sha1(b"other")}
+
+ result = self._run(
+ {"icon.png": b"new", "nested/icon.svg": b""}, digests=digests
+ )
+
+ self._assert_failed(result, "nested/icon.svg")
+ self.assertFalse((self.dest / "nested").exists())
+
+ def test_parent_reference_member_fails(self):
+ result = self._run(
+ {"icon.png": b"new", "evil": b"x"},
+ names={"evil": "../evil"},
+ digests={"icon.png": _sha1(b"new")},
+ )
+
+ self._assert_failed(result, "../evil")
+ self.assertFalse((self.root / "evil").exists())
+
+ def test_absolute_member_fails(self):
+ result = self._run(
+ {"icon.png": b"new", "evil": b"x"},
+ names={"evil": f"{self.root}/evil"},
+ digests={"icon.png": _sha1(b"new")},
+ )
+
+ self._assert_failed(result, "leaves the destination")
+ self.assertFalse((self.root / "evil").exists())
+
+ def test_unexpected_member_fails(self):
+ result = self._run(
+ {"icon.png": b"new", "extra.png": b"x"},
+ digests={"icon.png": _sha1(b"new")},
+ )
+
+ self._assert_failed(result, "extra.png")
+
+ def test_missing_member_fails(self):
+ result = self._run(
+ {"icon.png": b"new"},
+ digests={"icon.png": _sha1(b"new"), "gone.png": _sha1(b"x")},
+ )
+
+ self._assert_failed(result, "gone.png")
+
+ def test_link_member_fails(self):
+ result = self._run(
+ {"icon.png": b"new", "link.png": b""},
+ kinds={"link.png": tarfile.SYMTYPE},
+ )
+
+ self._assert_failed(result, "not a regular file")
+
+ def test_parent_reference_in_the_arguments_fails(self):
+ result = self._run({}, directories=["../outside"])
+
+ self.assertTrue(result["failed"])
+ self.assertIn("../outside", result["msg"])
+ self.assertFalse((self.root / "outside").exists())
+
+ def test_write_failure_names_the_file_and_fails(self):
+ original = AnsibleModule.atomic_move
+
+ def move(module, src, dest, **kwargs):
+ if dest.endswith("nested/icon.svg"):
+ raise Exception("Could not replace") # noqa: TRY002 - atomic_move raises a bare Exception
+ return original(module, src, dest, **kwargs)
+
+ with patch.object(
+ AnsibleModule, "atomic_move", autospec=True, side_effect=move
+ ):
+ result = self._run({"icon.png": b"new", "nested/icon.svg": b""})
+
+ self.assertTrue(result["failed"])
+ self.assertIn(str(self.dest / "nested/icon.svg"), result["msg"])
+ self.assertTrue(result["changed"])
+
+ def test_failure_after_creating_a_parent_reports_the_change(self):
+ with patch.object(
+ AnsibleModule,
+ "atomic_move",
+ autospec=True,
+ side_effect=Exception("Could not replace"),
+ ):
+ result = self._run({"nested/icon.svg": b""})
+
+ self.assertTrue(result["failed"])
+ self.assertTrue(result["changed"])
+ self.assertTrue((self.dest / "nested").is_dir())
+
+ def test_repeated_file_path_fails(self):
+ result = self._run(
+ {"fresh.png": b"fresh"},
+ digests={"fresh.png": _sha1(b"fresh")},
+ tree=["fresh.png"],
+ )
+ self.assertNotIn("failed", result)
+
+ repeated = _call(
+ _MOD.main,
+ {
+ "src": str(self.archive),
+ "dest": f"{self.dest}/",
+ "files": [
+ {"path": "fresh.png", "checksum": _sha1(b"fresh")},
+ {"path": "fresh.png", "checksum": _sha1(b"fresh")},
+ ],
+ "settled": [],
+ "directories": [],
+ "tree": ["fresh.png"],
+ "mode": "0644",
+ "directory_mode": None,
+ "_ansible_tmpdir": str(self.work),
+ },
+ )
+
+ self.assertTrue(repeated["failed"])
+ self.assertIn("repeat", repeated["msg"])
+
+
+class TestOrderAndAliases(_Tree):
+ def test_files_are_written_in_the_given_order(self):
+ order = ["b/z.png", "a.png", "c.png"]
+
+ with patch.object(
+ AnsibleModule,
+ "atomic_move",
+ autospec=True,
+ side_effect=AnsibleModule.atomic_move,
+ ) as move:
+ self._run({name: name.encode() for name in order})
+
+ self.assertEqual(
+ [call.args[2] for call in move.call_args_list],
+ [str(self.dest / name) for name in order],
+ )
+
+ def test_settled_files_are_handled_in_tree_order(self):
+ settled = self._write("s.png", b"s", 0o600)
+
+ with patch.object(
+ AnsibleModule,
+ "atomic_move",
+ autospec=True,
+ side_effect=Exception("Could not replace"),
+ ):
+ result = self._run(
+ {"b.png": b"b"}, settled=["s.png"], tree=["s.png", "b.png"]
+ )
+
+ self.assertTrue(result["failed"])
+ self.assertEqual(self._mode(settled), 0o644)
+
+ def test_settled_file_after_a_failed_write_stays_untouched(self):
+ settled = self._write("s.png", b"s", 0o600)
+
+ with patch.object(
+ AnsibleModule,
+ "atomic_move",
+ autospec=True,
+ side_effect=Exception("Could not replace"),
+ ):
+ result = self._run(
+ {"b.png": b"b"}, settled=["s.png"], tree=["b.png", "s.png"]
+ )
+
+ self.assertTrue(result["failed"])
+ self.assertEqual(self._mode(settled), 0o600)
+
+ def test_path_missing_from_tree_fails(self):
+ result = self._run({"b.png": b"b"}, tree=[])
+
+ self.assertTrue(result["failed"])
+ self.assertIn("missing from tree", result["msg"])
+
+ def test_two_paths_through_a_symlinked_directory_are_refused(self):
+ (self.dest / "y").mkdir()
+ self._write("y/f.js", b"y", 0o644)
+ (self.dest / "x").symlink_to(self.dest / "y")
+
+ result = self._run(
+ {"x/f.js": b"x", "fresh.png": b"fresh"},
+ tree=["x/f.js", "y/f.js", "fresh.png"],
+ )
+
+ self.assertFalse(result["changed"])
+ self.assertIn("reach the same entry", " ".join(result["refused"]))
+ self.assertEqual(self._bytes(self.dest / "y/f.js"), b"y")
+ self.assertFalse((self.dest / "fresh.png").exists())
+
+ def test_two_missing_paths_through_a_symlinked_directory_are_refused(self):
+ (self.dest / "y").mkdir()
+ (self.dest / "x").symlink_to(self.dest / "y")
+
+ result = self._run({"x/f.js": b"x", "y/f.js": b"y"}, tree=["x/f.js", "y/f.js"])
+
+ self.assertIn("reach the same entry", " ".join(result["refused"]))
+ self.assertFalse((self.dest / "y/f.js").exists())
+
+ def test_hard_linked_targets_are_refused(self):
+ first = self._write("a.png", b"same", 0o644)
+ (self.dest / "b.png").hardlink_to(first)
+
+ result = self._run({"a.png": b"new"}, tree=["a.png", "b.png"])
+
+ self.assertIn("reach the same entry", " ".join(result["refused"]))
+ self.assertEqual(self._bytes(self.dest / "b.png"), b"same")
+
+ def test_unsafe_writes_reaches_atomic_move_from_the_environment(self):
+ with (
+ patch.dict(os.environ, {"ANSIBLE_UNSAFE_WRITES": "1"}),
+ patch.object(
+ AnsibleModule,
+ "atomic_move",
+ autospec=True,
+ side_effect=AnsibleModule.atomic_move,
+ ) as move,
+ ):
+ self._run({"icon.png": b"new"})
+
+ self.assertTrue(move.call_args.kwargs["unsafe_writes"])
+
+ def test_leaf_created_concurrently_is_accepted(self):
+ original = Path.mkdir
+
+ def racing(path, *args, **kwargs):
+ original(path, *args, **kwargs)
+ raise FileExistsError(path)
+
+ with patch.object(Path, "mkdir", autospec=True, side_effect=racing):
+ result = self._run({}, directories=["leaf"], directory_mode="0750")
+
+ self.assertTrue(result["changed"])
+ self.assertEqual(self._mode(self.dest / "leaf"), 0o750)
+
+
+class TestSameEndStateAsTheCopyModule(unittest.TestCase):
+ MODE = "0644"
+ DIRECTORY_MODE = "0750"
+ CONTENT: ClassVar[dict[str, bytes]] = {
+ "replaced.png": b"new",
+ "fresh/deep/new.svg": b"",
+ "kept-dir/new.png": b"png",
+ }
+ SETTLED: ClassVar[tuple[str, ...]] = ("mode-drift.png",)
+ LEAVES: ClassVar[tuple[str, ...]] = ("existing-leaf", "absent/leaf")
+
+ def setUp(self):
+ self._tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self._tmp.name)
+
+ def tearDown(self):
+ self._tmp.cleanup()
+
+ def _seed(self, dest):
+ dest.mkdir()
+ (dest / "replaced.png").write_bytes(b"old")
+ (dest / "replaced.png").chmod(0o600)
+ (dest / "mode-drift.png").write_bytes(b"same")
+ (dest / "mode-drift.png").chmod(0o600)
+ (dest / "kept-dir").mkdir()
+ (dest / "kept-dir").chmod(0o701)
+ (dest / "existing-leaf").mkdir()
+ (dest / "existing-leaf").chmod(0o700)
+ (dest / "untouched.png").write_bytes(b"untouched")
+
+ def _regular_copy(self, dest):
+ staging = self.root / "staging"
+ staging.mkdir()
+ for rel, data in self.CONTENT.items():
+ source = staging / ".source"
+ source.write_bytes(data)
+ result = _call(
+ copy_module.main,
+ {
+ "src": str(source),
+ "dest": f"{dest}/",
+ "mode": self.MODE,
+ "directory_mode": self.DIRECTORY_MODE,
+ "_original_basename": rel,
+ "follow": False,
+ "checksum": _sha1(data),
+ },
+ )
+ self.assertNotIn("failed", result, result)
+ for rel in self.SETTLED:
+ result = _call(
+ file_module.main,
+ {
+ "mode": self.MODE,
+ "dest": f"{dest}/",
+ "_original_basename": rel,
+ "recurse": False,
+ "state": "file",
+ },
+ )
+ self.assertNotIn("failed", result, result)
+ for rel in self.LEAVES:
+ result = _call(
+ file_module.main,
+ {
+ "path": str(dest / rel),
+ "state": "directory",
+ "mode": self.DIRECTORY_MODE,
+ "recurse": False,
+ },
+ )
+ self.assertNotIn("failed", result, result)
+
+ def _batch(self, dest):
+ archive = self.root / "archive.tar"
+ work = self.root / "work"
+ work.mkdir()
+ with tarfile.open(archive, "w") as bundle:
+ for rel, data in self.CONTENT.items():
+ member = tarfile.TarInfo(rel)
+ member.size = len(data)
+ bundle.addfile(member, io.BytesIO(data))
+ result = _call(
+ _MOD.main,
+ {
+ "src": str(archive),
+ "dest": f"{dest}/",
+ "files": [
+ {"path": rel, "checksum": _sha1(data)}
+ for rel, data in self.CONTENT.items()
+ ],
+ "settled": list(self.SETTLED),
+ "directories": list(self.LEAVES),
+ "tree": [*self.CONTENT, *self.SETTLED, *self.LEAVES],
+ "mode": self.MODE,
+ "directory_mode": self.DIRECTORY_MODE,
+ "_ansible_tmpdir": str(work),
+ },
+ )
+ self.assertTrue(result["changed"], result)
+
+ @staticmethod
+ def _snapshot(dest):
+ entries = {}
+ for path in sorted(dest.rglob("*")): # nocheck: project-walk
+ info = path.lstat()
+ content = None
+ if path.is_file():
+ content = path.read_bytes() # nocheck: cache-read
+ entries[str(path.relative_to(dest))] = (
+ stat.S_IFMT(info.st_mode),
+ stat.S_IMODE(info.st_mode),
+ info.st_uid,
+ info.st_gid,
+ content,
+ )
+ return entries
+
+ def test_batch_leaves_the_tree_the_regular_copy_leaves(self):
+ regular, batch = self.root / "regular", self.root / "batch"
+ self._seed(regular)
+ self._seed(batch)
+
+ self._regular_copy(regular)
+ self._batch(batch)
+
+ self.assertEqual(self._snapshot(batch), self._snapshot(regular))
+ self.assertEqual(self._snapshot(batch)["kept-dir"][1], 0o701)
+ self.assertEqual(self._snapshot(batch)["existing-leaf"][1], 0o750)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/unit/python/plugins/action/test_stack_host_copy.py b/tests/unit/python/plugins/action/test_stack_host_copy.py
index de6440b8b5..eedb06da62 100644
--- a/tests/unit/python/plugins/action/test_stack_host_copy.py
+++ b/tests/unit/python/plugins/action/test_stack_host_copy.py
@@ -1,5 +1,6 @@
import hashlib
import os
+import tarfile
import tempfile
import unittest
from pathlib import Path
@@ -153,6 +154,7 @@ class _FakeShell:
class _FakeConnection:
def __init__(self):
self._shell = _FakeShell()
+ self.become = None
class _FakeLoader:
@@ -179,6 +181,11 @@ class _Dir:
self.mode = mode
+class _Link:
+ def __init__(self, data):
+ self.data = data
+
+
def _mode_bits(mode):
return mode if isinstance(mode, int) else int(mode, 8)
@@ -188,6 +195,24 @@ class _FakeRemote:
self.entries = dict(entries)
self.calls = []
self.args = []
+ self.transfers = []
+ self.fixups = 0
+
+ def transfer(self, local, remote_path):
+ if not remote_path.endswith(".tar"):
+ self.transfers.append(Path(local).read_bytes()) # nocheck: cache-read
+ return remote_path
+ with tarfile.open(local) as bundle:
+ self.transfers.append(
+ {
+ member.name: bundle.extractfile(member).read()
+ for member in bundle.getmembers()
+ }
+ )
+ return remote_path
+
+ def fixup(self, *args, **kwargs):
+ self.fixups += 1
def __call__(self, module_name=None, module_args=None, task_vars=None, **kwargs):
self.calls.append(module_name)
@@ -197,14 +222,31 @@ class _FakeRemote:
"changed": False,
"states": {p: self._state(p) for p in module_args["paths"]},
}
+ if module_name == "ansible.legacy.copy_batch":
+ return self._copy_batch(module_args)
if module_name == "ansible.legacy.stat":
entry = self.entries.get(module_args["path"])
if entry is None:
return {"stat": {"exists": False}}
if isinstance(entry, _Dir):
return {"stat": {"exists": True, "isdir": True}}
+ if isinstance(entry, _Link):
+ digest = _sha1(entry.data) if module_args["follow"] else ""
+ return {
+ "stat": {
+ "exists": True,
+ "isdir": False,
+ "islnk": True,
+ "checksum": digest,
+ }
+ }
return {
- "stat": {"exists": True, "isdir": False, "checksum": _sha1(entry[0])}
+ "stat": {
+ "exists": True,
+ "isdir": False,
+ "islnk": False,
+ "checksum": _sha1(entry[0]),
+ }
}
if module_name == "ansible.legacy.file" and module_args["state"] == "directory":
return self._ensure_directory(module_args["path"], module_args["mode"])
@@ -228,9 +270,39 @@ class _FakeRemote:
"path": path,
}
+ def _copy_batch(self, args):
+ dest = args["dest"]
+ files = [entry["path"] for entry in args["files"]]
+ refused = [
+ dest + rel
+ for rel in [*files, *args["settled"]]
+ if isinstance(self.entries.get(dest + rel), (_Dir, _Link))
+ ] + [
+ dest + rel
+ for rel in args["directories"]
+ if dest + rel in self.entries
+ and not isinstance(self.entries[dest + rel], _Dir)
+ ]
+ if refused:
+ return {"changed": False, "refused": refused}
+ archive = self.transfers[-1]
+ if list(archive) != files or any(
+ _sha1(archive[entry["path"]]) != entry["checksum"]
+ for entry in args["files"]
+ ):
+ return {"failed": True, "msg": "archive does not match files"}
+ mode = _mode_bits(args["mode"])
+ for rel in files:
+ self.entries[dest + rel] = (archive[rel], mode)
+ for rel in args["settled"]:
+ self.entries[dest + rel] = (self.entries[dest + rel][0], mode)
+ for rel in args["directories"]:
+ self.entries[dest + rel] = _Dir(_mode_bits(args["directory_mode"] or 0o755))
+ return {"changed": True}
+
def _state(self, path):
entry = self.entries.get(path)
- if entry is None:
+ if entry is None or isinstance(entry, _Link):
return {"isreg": False, "isdir": False, "mode": None, "checksum": None}
if isinstance(entry, _Dir):
return {
@@ -247,12 +319,8 @@ class _FakeRemote:
}
-class TestStackHostCopyTree(unittest.TestCase):
- FILES: ClassVar[dict[str, bytes]] = {
- "favicon.ico": b"ico",
- "icon.png": b"png",
- "nested/icon.svg": b"",
- }
+class _TreeHarness(unittest.TestCase):
+ FILES: ClassVar[dict[str, bytes]] = {}
DEST = "/var/www/public_cdn/roles/app/latest/img/design/"
def setUp(self):
@@ -268,15 +336,19 @@ class TestStackHostCopyTree(unittest.TestCase):
def _synced(self, mode=0o644):
return {self.DEST + rel: (data, mode) for rel, data in self.FILES.items()}
- def _run(self, remote, **extra):
+ def _run(self, remote, *, check_mode=False, diff=False, loader=None, **extra):
task = _FakeTask(args={"src": f"{self.src}/", "dest": self.DEST, **extra})
+ task.check_mode = check_mode
+ task.diff = diff
action = _make_action(task, _FakeTemplar(is_stack_host=True))
action._connection = _FakeConnection()
- action._loader = _FakeLoader()
+ action._loader = _FakeLoader() if loader is None else loader
action._play_context = _FakePlayContext()
action._execute_module = remote
- action._transfer_file = lambda local, remote_path: remote_path
- action._fixup_perms2 = lambda *a, **k: None
+ action._transfer_file = getattr(
+ remote, "transfer", lambda local, remote_path: remote_path
+ )
+ action._fixup_perms2 = getattr(remote, "fixup", lambda *a, **k: None)
action._remove_tmp_path = lambda *a, **k: None
action._remote_expand_user = lambda path, **k: path
@@ -287,6 +359,36 @@ class TestStackHostCopyTree(unittest.TestCase):
action._make_tmp_path = make_tmp_path
return action.run(task_vars={})
+ BATCH: ClassVar[list[str]] = [
+ "ansible.legacy.file_checksums",
+ "ansible.legacy.copy_batch",
+ ]
+
+ @staticmethod
+ def _batch_args(remote):
+ return remote.args[remote.calls.index("ansible.legacy.copy_batch")]
+
+ @staticmethod
+ def _copies(remote):
+ return [
+ args
+ for name, args in zip(remote.calls, remote.args, strict=True)
+ if name == "ansible.legacy.copy"
+ ]
+
+
+class TestStackHostCopyTree(_TreeHarness):
+ FILES: ClassVar[dict[str, bytes]] = {
+ "favicon.ico": b"ico",
+ "icon.png": b"png",
+ "nested/icon.svg": b"",
+ }
+
+ def _changed(self):
+ entries = self._synced()
+ entries[self.DEST + "icon.png"] = (b"old", 0o644)
+ return entries
+
def test_unchanged_tree_costs_one_remote_call(self):
remote = _FakeRemote(self._synced())
@@ -297,7 +399,7 @@ class TestStackHostCopyTree(unittest.TestCase):
self.assertEqual(result["dest"], self.DEST)
self.assertEqual(result["src"], f"{self.src}/")
- def test_changed_file_runs_the_regular_copy(self):
+ def test_changed_file_travels_alone_in_one_batch(self):
entries = self._synced()
entries[self.DEST + "icon.png"] = (b"old", 0o644)
remote = _FakeRemote(entries)
@@ -305,9 +407,16 @@ class TestStackHostCopyTree(unittest.TestCase):
result = self._run(remote)
self.assertTrue(result["changed"])
- self.assertIn("ansible.legacy.copy", remote.calls)
+ self.assertEqual(remote.calls, self.BATCH)
+ self.assertEqual(remote.transfers, [{"icon.png": b"png"}])
+ self.assertEqual(
+ self._batch_args(remote)["files"],
+ [{"path": "icon.png", "checksum": _sha1(b"png")}],
+ )
+ self.assertEqual(sorted(self._batch_args(remote)["tree"]), sorted(self.FILES))
+ self.assertEqual(remote.entries, self._synced())
- def test_missing_file_runs_the_regular_copy(self):
+ def test_missing_file_travels_in_one_batch(self):
entries = self._synced()
del entries[self.DEST + "nested/icon.svg"]
remote = _FakeRemote(entries)
@@ -315,9 +424,10 @@ class TestStackHostCopyTree(unittest.TestCase):
result = self._run(remote)
self.assertTrue(result["changed"])
- self.assertIn("ansible.legacy.copy", remote.calls)
+ self.assertEqual(remote.calls, self.BATCH)
+ self.assertEqual(remote.transfers, [{"nested/icon.svg": b""}])
- def test_mode_drift_runs_the_regular_copy(self):
+ def test_mode_drift_is_settled_without_sending_the_file(self):
entries = self._synced()
entries[self.DEST + "favicon.ico"] = (b"ico", 0o600)
remote = _FakeRemote(entries)
@@ -325,16 +435,165 @@ class TestStackHostCopyTree(unittest.TestCase):
result = self._run(remote)
self.assertTrue(result["changed"])
- self.assertIn("ansible.legacy.file", remote.calls)
+ self.assertEqual(remote.calls, self.BATCH)
+ self.assertEqual(remote.transfers, [{}])
+ self.assertEqual(self._batch_args(remote)["settled"], ["favicon.ico"])
+ self.assertEqual(self._batch_args(remote)["files"], [])
- def test_directory_in_place_of_a_file_runs_the_regular_copy(self):
+ def test_directory_in_place_of_a_file_is_refused_then_copied_regularly(self):
entries = self._synced()
entries[self.DEST + "icon.png"] = _Dir(0o755)
remote = _FakeRemote(entries)
self._run(remote)
+ self.assertEqual(remote.calls[:2], self.BATCH)
self.assertIn("ansible.legacy.stat", remote.calls)
+ self.assertIn("ansible.legacy.copy", remote.calls)
+
+ def test_symlink_in_place_of_a_file_is_refused_then_copied_regularly(self):
+ entries = self._synced()
+ entries[self.DEST + "icon.png"] = _Link(b"png")
+ remote = _FakeRemote(entries)
+
+ self._run(remote)
+
+ self.assertEqual(remote.calls[:2], self.BATCH)
+ copies = self._copies(remote)
+ self.assertEqual([args["_original_basename"] for args in copies], ["icon.png"])
+ self.assertFalse(copies[0]["follow"])
+
+ def test_check_mode_keeps_the_regular_copy(self):
+ remote = _FakeRemote(self._changed())
+
+ result = self._run(remote, check_mode=True)
+
+ self.assertTrue(result["changed"])
+ self.assertNotIn("ansible.legacy.copy_batch", remote.calls)
+ self.assertNotIn("ansible.legacy.copy", remote.calls)
+ self.assertIn("ansible.legacy.stat", remote.calls)
+ self.assertEqual(remote.transfers, [])
+
+ def test_diff_mode_keeps_the_regular_copy(self):
+ remote = _FakeRemote(self._changed())
+
+ with patch(
+ "plugins.action.stack_host_copy.CopyActionModule.run",
+ return_value={"changed": True, "diff": [{"after": "png"}]},
+ ) as super_run:
+ result = self._run(remote, diff=True)
+
+ super_run.assert_called_once()
+ self.assertEqual(result["diff"], [{"after": "png"}])
+ self.assertEqual(remote.calls, ["ansible.legacy.file_checksums"])
+ self.assertEqual(remote.transfers, [])
+
+ def test_failed_batch_stays_failed_without_a_second_copy(self):
+ failure = {
+ "failed": True,
+ "changed": True,
+ "msg": f"Failed to write {self.DEST}icon.png: boom",
+ }
+
+ class _FailingBatch(_FakeRemote):
+ def _copy_batch(self, args):
+ return dict(failure)
+
+ remote = _FailingBatch(self._changed())
+
+ with patch("plugins.action.stack_host_copy.CopyActionModule.run") as super_run:
+ result = self._run(remote)
+
+ super_run.assert_not_called()
+ self.assertTrue(result["failed"])
+ self.assertIn(f"{self.DEST}icon.png", result["msg"])
+ self.assertEqual(remote.calls, self.BATCH)
+
+ def test_batch_result_has_the_keys_of_the_regular_copy(self):
+ batch = self._run(_FakeRemote(self._changed()))
+ with patch.object(ActionModule, "_write_tree", return_value=None):
+ regular = self._run(_FakeRemote(self._changed()))
+
+ self.assertEqual(set(batch), set(regular))
+ self.assertEqual(
+ {key: batch[key] for key in ("changed", "dest", "src")},
+ {key: regular[key] for key in ("changed", "dest", "src")},
+ )
+
+ def test_unprivileged_become_keeps_the_regular_copy(self):
+ remote = _FakeRemote(self._changed())
+
+ with patch.object(ActionModule, "_is_become_unprivileged", return_value=True):
+ result = self._run(remote)
+
+ self.assertTrue(result["changed"])
+ self.assertNotIn("ansible.legacy.copy_batch", remote.calls)
+ self.assertIn("ansible.legacy.copy", remote.calls)
+
+ def test_dest_with_a_variable_skips_the_tree_check(self):
+ remote = _FakeRemote(self._changed())
+ task = _FakeTask(args={"src": f"{self.src}/", "dest": "/srv/$APP/img/"})
+ action = _make_action(task, _FakeTemplar(is_stack_host=True))
+ action._execute_module = remote
+
+ with patch(
+ "plugins.action.stack_host_copy.CopyActionModule.run",
+ return_value={"changed": True},
+ ) as super_run:
+ action.run(task_vars={})
+
+ super_run.assert_called_once()
+ self.assertEqual(remote.calls, [])
+
+ def test_archive_over_the_limit_keeps_the_regular_copy(self):
+ entries = self._synced()
+ entries[self.DEST + "icon.png"] = (b"old", 0o644)
+ entries[self.DEST + "favicon.ico"] = (b"old", 0o644)
+ remote = _FakeRemote(entries)
+
+ with patch("plugins.action.stack_host_copy._ARCHIVE_LIMIT", 5):
+ result = self._run(remote)
+
+ self.assertTrue(result["changed"])
+ self.assertNotIn("ansible.legacy.copy_batch", remote.calls)
+ self.assertEqual(sorted(remote.transfers), [b"ico", b"png"])
+
+ def test_files_travel_in_the_order_the_copy_action_walks(self):
+ remote = _FakeRemote({})
+
+ self._run(remote)
+
+ walked = [
+ rel
+ for _, rel in _walk_dirs_without_bytecode_caches(
+ f"{self.src}/", local_follow=True
+ )["files"]
+ ]
+ sent = [entry["path"] for entry in self._batch_args(remote)["files"]]
+ self.assertEqual(sent, walked)
+ self.assertEqual(list(remote.transfers[0]), walked)
+
+ def test_checksum_is_taken_from_the_packed_bytes(self):
+ remote = _FakeRemote(self._changed())
+ icon = self.src / "icon.png"
+ loader = _FakeLoader()
+ reads = []
+
+ def get_real_file(path, decrypt=True):
+ reads.append(path)
+ if path == str(icon) and reads.count(path) == 2:
+ icon.write_bytes(b"edited")
+ return path
+
+ loader.get_real_file = get_real_file
+
+ self._run(remote, loader=loader)
+
+ self.assertEqual(remote.transfers, [{"icon.png": b"edited"}])
+ self.assertEqual(
+ self._batch_args(remote)["files"],
+ [{"path": "icon.png", "checksum": _sha1(b"edited")}],
+ )
def test_requested_mode_is_what_the_tree_check_compares(self):
remote = _FakeRemote(self._synced(mode=0o600))
@@ -437,7 +696,7 @@ class TestStackHostCopyTree(unittest.TestCase):
regular = _FakeRemote(self._with_leaves(0o700))
settled = self._run(remote)
- with patch.object(ActionModule, "_unchanged_tree", return_value=None):
+ with patch.object(ActionModule, "_settled_tree", return_value=None):
copied = self._run(regular)
self.assertFalse(settled["changed"])
@@ -454,7 +713,7 @@ class TestStackHostCopyTree(unittest.TestCase):
self.assertFalse(result["changed"])
self.assertEqual(remote.calls, ["ansible.legacy.file_checksums"])
- def test_missing_remote_directory_runs_the_regular_copy(self):
+ def test_missing_remote_directory_is_ensured_in_one_batch(self):
self._add_empty_directories()
entries = self._with_leaves(0o755)
del entries[self.DEST + "nested/.claude/.cc-writes"]
@@ -463,18 +722,24 @@ class TestStackHostCopyTree(unittest.TestCase):
result = self._run(remote)
self.assertTrue(result["changed"])
- self.assertEqual(self._directory_calls(remote), self._leaf_paths())
+ self.assertEqual(remote.calls, self.BATCH)
+ self.assertEqual(
+ self._batch_args(remote)["directories"], ["nested/.claude/.cc-writes"]
+ )
+ self.assertIsNone(self._batch_args(remote)["directory_mode"])
- def test_directory_mode_drift_runs_the_regular_copy(self):
+ def test_directory_mode_drift_is_settled_in_one_batch(self):
self._add_empty_directories()
remote = _FakeRemote(self._with_leaves(0o755))
result = self._run(remote, directory_mode="0750")
self.assertTrue(result["changed"])
- self.assertEqual(self._directory_calls(remote), self._leaf_paths())
+ self.assertEqual(remote.calls, self.BATCH)
+ self.assertEqual(self._batch_args(remote)["directories"], sorted(self.LEAVES))
+ self.assertEqual(self._batch_args(remote)["directory_mode"], "0750")
- def test_file_in_place_of_a_directory_runs_the_regular_copy(self):
+ def test_file_in_place_of_a_directory_is_refused_then_fails_regularly(self):
self._add_empty_directories()
entries = self._with_leaves(0o755)
entries[self.DEST + ".claude/.cc-writes"] = (b"", 0o644)
@@ -482,6 +747,7 @@ class TestStackHostCopyTree(unittest.TestCase):
result = self._run(remote)
+ self.assertEqual(remote.calls[:2], self.BATCH)
self.assertTrue(result["failed"])
self.assertIn(self.DEST + ".claude/.cc-writes", self._directory_calls(remote))
@@ -541,9 +807,10 @@ class TestStackHostCopyTree(unittest.TestCase):
result = self._run(remote)
self.assertTrue(result["changed"])
- self.assertIn("ansible.legacy.copy", remote.calls)
+ self.assertEqual(remote.calls, self.BATCH)
touched = [str(value) for args in remote.args for value in args.values()]
self.assertEqual([value for value in touched if "__pycache__" in value], [])
+ self.assertEqual(remote.transfers, [{"icon.png": b"png"}])
def test_bytecode_outside_a_cache_directory_stays_in_the_copy(self):
(self.src / "legacy.pyc").write_bytes(b"bytecode")
@@ -553,7 +820,7 @@ class TestStackHostCopyTree(unittest.TestCase):
self.assertTrue(result["changed"])
self.assertIn(self.DEST + "legacy.pyc", remote.args[0]["paths"])
- self.assertIn("ansible.legacy.copy", remote.calls)
+ self.assertEqual(remote.transfers, [{"legacy.pyc": b"bytecode"}])
def test_symlinked_root_passes_the_cache_filter_unchanged(self):
with tempfile.TemporaryDirectory() as links:
@@ -565,6 +832,79 @@ class TestStackHostCopyTree(unittest.TestCase):
self.assertEqual(filtered["symlinks"], (str(self.src), "root"))
+class TestStackHostCopyFlatTree(_TreeHarness):
+ FILES: ClassVar[dict[str, bytes]] = {
+ f"slot-{index:02d}.png": f"palette-2-{index}".encode() for index in range(15)
+ }
+
+ def _stale(self, names):
+ entries = self._synced()
+ entries.update({self.DEST + name: (b"palette-1", 0o644) for name in names})
+ return entries
+
+ def _round_trips(self, entries, *, regular):
+ remote = _FakeRemote(entries)
+ if regular:
+ with patch.object(ActionModule, "_write_tree", return_value=None):
+ result = self._run(remote)
+ else:
+ result = self._run(remote)
+ self.assertTrue(result["changed"])
+ return {
+ "modules": len(remote.calls),
+ "transfers": len(remote.transfers),
+ "fixups": remote.fixups,
+ }
+
+ def test_all_files_changed_cost_one_transfer_and_two_module_calls(self):
+ stale = self._stale(self.FILES)
+
+ before = self._round_trips(stale, regular=True)
+ after = self._round_trips(stale, regular=False)
+
+ self.assertEqual(before, {"modules": 31, "transfers": 15, "fixups": 15})
+ self.assertEqual(after, {"modules": 2, "transfers": 1, "fixups": 1})
+
+ def test_one_changed_file_costs_one_transfer_and_two_module_calls(self):
+ stale = self._stale(["slot-07.png"])
+
+ before = self._round_trips(stale, regular=True)
+ after = self._round_trips(stale, regular=False)
+
+ self.assertEqual(before, {"modules": 31, "transfers": 1, "fixups": 1})
+ self.assertEqual(after, {"modules": 2, "transfers": 1, "fixups": 1})
+
+ def test_only_the_changed_file_is_sent(self):
+ remote = _FakeRemote(self._stale(["slot-07.png"]))
+
+ self._run(remote)
+
+ self.assertEqual(remote.transfers, [{"slot-07.png": b"palette-2-7"}])
+ self.assertEqual(remote.entries, self._synced())
+
+ def test_follow_without_symlinks_takes_the_batch(self):
+ remote = _FakeRemote(self._stale(["slot-00.png", "slot-01.png"]))
+
+ result = self._run(remote, follow=True)
+
+ self.assertTrue(result["changed"])
+ self.assertEqual(remote.calls, self.BATCH)
+
+ def test_follow_onto_a_symlinked_file_keeps_the_copy_semantics(self):
+ entries = self._synced()
+ entries[self.DEST + "slot-00.png"] = _Link(b"elsewhere")
+ remote = _FakeRemote(entries)
+
+ self._run(remote, follow=True)
+
+ self.assertEqual(remote.calls[:2], self.BATCH)
+ copies = self._copies(remote)
+ self.assertEqual(
+ [args["_original_basename"] for args in copies], ["slot-00.png"]
+ )
+ self.assertTrue(copies[0]["follow"])
+
+
class TestStackHostCopyWalkSwap(unittest.TestCase):
def _action(self):
task = _FakeTask(args={"src": "x", "dest": "/opt/compose/app/x"})