mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-10 12:47:04 +00:00
perf(stack-host-copy): write a changed tree in one transfer
When the tree check finds differing entries, pack exactly the differing files into one tar archive in source walk order, transfer it once into the remote tmpdir and let the new copy_batch module write each file through atomic_move with the copy module's directory, mode and unsafe_writes semantics. Mode-only drift and leaf directories settle in the same call; unchanged files are not sent. Every shape the regular copy treats differently makes the module refuse before it writes: a symlink, directory or other non-regular entry at a destination, a missing path below a non-directory or a symlink loop, and two source paths reaching one remote entry. The regular copy then runs. Check mode, diff mode, unprivileged become, a dest holding $ and an archive above 16 MiB keep the regular copy. A write failure fails the task and names the file. Executed with ansible-playbook -c local, pipelining on and off: the trees this action leaves equal the builtin copy's, including the symlink fallback; a 6-file changed tree took 2 module calls and 1 transfer instead of 14 and 4. Unit: a flat 15-file tree with every file changed drops from 31 module calls and 15 transfers to 2 and 1. Two dialectic rounds with three independent skeptics; every hole they found is fixed and tested. A third independent review compared end state, changed, check and diff mode against the builtin copy in a real playbook run and found them equal; one deliberate difference remains: a write failing partway reports changed=true with the files already written, where the builtin copy reports changed=false. | 🎯 Fix | 💥 Break | 🌱 Efficiency | ⚡ Speed | 🛡️ Security | 🏆 Score | Gate | | --- | --- | --- | --- | --- | --- | --- | | 90 % | 2 % | 100 % | 15 % (estimated from round trips) | ~100 % | 100 | ⏳ score passes, gate.py not run | Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
025ebe9754
commit
a643871997
4 files changed
+1574
-50
No files matched your search
@@ -0,0 +1,332 @@
|
||||
"""Write the differing entries of a directory copy in one module call.
|
||||
|
||||
Every path is handled like ``ansible.builtin.copy`` handles one entry of a
|
||||
recursive copy: a missing parent directory is created and receives
|
||||
``directory_mode`` while an existing one stays as found, a file is replaced
|
||||
through ``atomic_move`` and receives ``mode``, and a leaf directory is ensured
|
||||
like ``ansible.builtin.file`` with ``state: directory``. Files are handled in
|
||||
the order of ``tree``, which is the order the copy action walks the source
|
||||
in, and leaf directories after them.
|
||||
Nothing is written when a file path exists as anything but a regular file, a
|
||||
leaf directory path as anything but a directory, a missing path lies below
|
||||
anything but a directory, or two paths of ``tree`` name the same remote
|
||||
entry; the call then only reports ``refused``.
|
||||
|
||||
Args:
|
||||
src: tar archive holding exactly one regular member per entry of ``files``.
|
||||
dest: absolute destination directory.
|
||||
files: list of ``{path, checksum}`` in source walk order, ``path``
|
||||
relative to ``dest`` and ``checksum`` the SHA-1 of its archive member.
|
||||
settled: relative paths of regular files whose content is in place; only
|
||||
``mode`` is enforced on them.
|
||||
directories: relative paths of leaf directories to ensure.
|
||||
tree: relative path of every file and leaf directory of the source tree,
|
||||
in walk order; holds every path of ``files``, ``settled`` and
|
||||
``directories``.
|
||||
mode: mode of every path in ``files`` and ``settled``.
|
||||
directory_mode: mode of every directory this call creates and of every
|
||||
path in ``directories``; ``None`` keeps the mode the umask gives.
|
||||
unsafe_writes: the file common argument of the same name, handed to
|
||||
``atomic_move`` as the copy module does.
|
||||
|
||||
Returns:
|
||||
changed: whether any file, mode or directory changed.
|
||||
refused: the reasons nothing was written; present only then.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
from ansible.module_utils.basic import FILE_COMMON_ARGUMENTS, AnsibleModule
|
||||
|
||||
ARGUMENT_SPEC = {
|
||||
"src": {"type": "path", "required": True},
|
||||
"dest": {"type": "path", "required": True},
|
||||
"files": {
|
||||
"type": "list",
|
||||
"elements": "dict",
|
||||
"required": True,
|
||||
"options": {
|
||||
"path": {"type": "str", "required": True},
|
||||
"checksum": {"type": "str", "required": True},
|
||||
},
|
||||
},
|
||||
"settled": {"type": "list", "elements": "str", "required": True},
|
||||
"directories": {"type": "list", "elements": "str", "required": True},
|
||||
"tree": {"type": "list", "elements": "str", "required": True},
|
||||
"mode": {"type": "raw", "required": True},
|
||||
"directory_mode": {"type": "raw"},
|
||||
"unsafe_writes": FILE_COMMON_ARGUMENTS["unsafe_writes"],
|
||||
}
|
||||
|
||||
_CHUNK = 1 << 20
|
||||
|
||||
|
||||
def is_relative(name):
|
||||
"""Tell whether *name* is a normalised path strictly below the destination.
|
||||
|
||||
Args:
|
||||
name: relative path from the module arguments or the archive.
|
||||
"""
|
||||
path = PurePosixPath(name)
|
||||
return (
|
||||
bool(path.parts)
|
||||
and str(path) == name
|
||||
and not path.is_absolute()
|
||||
and ".." not in path.parts
|
||||
)
|
||||
|
||||
|
||||
def blocker(path, is_kind, kind, may_be_missing):
|
||||
"""Name what keeps *path* from taking an entry of the wanted type.
|
||||
|
||||
Args:
|
||||
path: absolute destination path.
|
||||
is_kind: ``stat.S_ISREG`` or ``stat.S_ISDIR``; an existing entry must
|
||||
pass it, so a symlink never does.
|
||||
kind: the wanted type in words.
|
||||
may_be_missing: whether a missing *path* below a directory is fine.
|
||||
|
||||
Returns:
|
||||
``None`` when *path* can take the entry, else the reason it cannot.
|
||||
"""
|
||||
try:
|
||||
info = path.lstat()
|
||||
except FileNotFoundError:
|
||||
if not may_be_missing:
|
||||
return f"{path} is missing"
|
||||
parent = path.parent
|
||||
while not os.path.lexists(parent):
|
||||
parent = parent.parent
|
||||
return None if parent.is_dir() else f"{parent} is not a directory"
|
||||
except OSError as error:
|
||||
return f"{path}: {error.strerror}"
|
||||
return None if is_kind(info.st_mode) else f"{path} is not {kind}"
|
||||
|
||||
|
||||
def aliases(dest, tree):
|
||||
"""Name the paths of *tree* that reach the same remote entry.
|
||||
|
||||
Args:
|
||||
dest: the destination directory.
|
||||
tree: relative paths of every file and leaf directory of the source.
|
||||
|
||||
Returns:
|
||||
One reason per path that reaches an entry an earlier path reaches.
|
||||
"""
|
||||
seen = {}
|
||||
reasons = []
|
||||
for name in tree:
|
||||
path = dest / name
|
||||
try:
|
||||
info = path.stat()
|
||||
key = (info.st_dev, info.st_ino)
|
||||
except OSError:
|
||||
key = str(Path(os.path.realpath(path.parent)) / path.name)
|
||||
if key in seen:
|
||||
reasons.append(f"{dest / seen[key]} and {path} reach the same entry")
|
||||
else:
|
||||
seen[key] = name
|
||||
return reasons
|
||||
|
||||
|
||||
def refusals(dest, files, settled, directories, tree):
|
||||
"""Collect what keeps any path of the call from taking its entry.
|
||||
|
||||
Args:
|
||||
dest: the destination directory.
|
||||
files: relative paths of the files to write.
|
||||
settled: relative paths of the files to keep.
|
||||
directories: relative paths of the leaf directories to ensure.
|
||||
tree: relative paths of every file and leaf directory of the source.
|
||||
|
||||
Returns:
|
||||
The reasons, empty when every path can take its entry.
|
||||
"""
|
||||
checks = [
|
||||
*((name, stat.S_ISREG, "a regular file", True) for name in files),
|
||||
*((name, stat.S_ISREG, "a regular file", False) for name in settled),
|
||||
*((name, stat.S_ISDIR, "a directory", True) for name in directories),
|
||||
]
|
||||
reasons = [
|
||||
blocker(dest / name, is_kind, kind, may_be_missing)
|
||||
for name, is_kind, kind, may_be_missing in checks
|
||||
]
|
||||
return [reason for reason in reasons if reason] + aliases(dest, tree)
|
||||
|
||||
|
||||
def members(module, archive, expected):
|
||||
"""Map every path of *expected* to its archive member, failing on any other.
|
||||
|
||||
Args:
|
||||
module: the running module.
|
||||
archive: the open tar archive.
|
||||
expected: mapping of relative path to the SHA-1 of its member.
|
||||
|
||||
Returns:
|
||||
Mapping of relative path to its tar member.
|
||||
"""
|
||||
found = {}
|
||||
for member in archive.getmembers():
|
||||
name = member.name
|
||||
if not is_relative(name):
|
||||
module.fail_json(msg=f"archive member {name!r} leaves the destination")
|
||||
if name not in expected or name in found:
|
||||
module.fail_json(msg=f"archive member {name!r} is unexpected or repeated")
|
||||
if not member.isreg():
|
||||
module.fail_json(msg=f"archive member {name!r} is not a regular file")
|
||||
found[name] = member
|
||||
missing = sorted(set(expected) - set(found))
|
||||
if missing:
|
||||
module.fail_json(msg=f"archive lacks {', '.join(missing)}")
|
||||
return found
|
||||
|
||||
|
||||
def sha1(archive, member):
|
||||
"""Return the SHA-1 of the content of *member*.
|
||||
|
||||
Args:
|
||||
archive: the open tar archive.
|
||||
member: a regular tar member.
|
||||
"""
|
||||
digest = hashlib.sha1(usedforsecurity=False)
|
||||
with archive.extractfile(member) as handle:
|
||||
while chunk := handle.read(_CHUNK):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def attributes(module, path, mode, changed):
|
||||
"""Apply *mode* to *path* like the copy and file modules do.
|
||||
|
||||
Args:
|
||||
module: the running module.
|
||||
path: the file or directory.
|
||||
mode: the wanted mode, or ``None`` to leave it.
|
||||
changed: the change state so far.
|
||||
|
||||
Returns:
|
||||
The change state including this call.
|
||||
"""
|
||||
file_args = module.load_file_common_arguments({"path": str(path), "mode": mode})
|
||||
return module.set_fs_attributes_if_different(file_args, changed)
|
||||
|
||||
|
||||
def make_directories(module, directory, directory_mode):
|
||||
"""Create *directory* and its missing parents, each with *directory_mode*.
|
||||
|
||||
Args:
|
||||
module: the running module.
|
||||
directory: the directory that must exist.
|
||||
directory_mode: mode of every directory created, or ``None``.
|
||||
|
||||
Returns:
|
||||
Whether a directory was missing.
|
||||
"""
|
||||
missing = []
|
||||
while not os.path.lexists(directory):
|
||||
missing.append(directory)
|
||||
directory = directory.parent
|
||||
for path in reversed(missing):
|
||||
try:
|
||||
path.mkdir()
|
||||
except FileExistsError:
|
||||
if not path.is_dir():
|
||||
raise
|
||||
attributes(module, path, directory_mode, True)
|
||||
return bool(missing)
|
||||
|
||||
|
||||
def write(module, archive, member, target, workdir):
|
||||
"""Replace *target* atomically with the content of *member*.
|
||||
|
||||
Args:
|
||||
module: the running module, whose ``atomic_move`` keeps the owner,
|
||||
group and SELinux context of a replaced file.
|
||||
archive: the open tar archive.
|
||||
member: the tar member holding the new content.
|
||||
target: destination path.
|
||||
workdir: directory the content is staged in before the move.
|
||||
"""
|
||||
descriptor, staged = tempfile.mkstemp(dir=workdir)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as out, archive.extractfile(member) as data:
|
||||
shutil.copyfileobj(data, out)
|
||||
module.atomic_move(
|
||||
staged, str(target), unsafe_writes=module.params["unsafe_writes"]
|
||||
)
|
||||
finally:
|
||||
Path(staged).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main():
|
||||
module = AnsibleModule(argument_spec=ARGUMENT_SPEC)
|
||||
params = module.params
|
||||
dest = Path(params["dest"])
|
||||
files = [entry["path"] for entry in params["files"]]
|
||||
expected = {entry["path"]: entry["checksum"] for entry in params["files"]}
|
||||
settled, directories = params["settled"], params["directories"]
|
||||
kept = set(settled)
|
||||
directory_mode = params["directory_mode"]
|
||||
names = [*files, *settled, *directories, *params["tree"]]
|
||||
unsafe = sorted({name for name in names if not is_relative(name)})
|
||||
if unsafe:
|
||||
module.fail_json(msg=f"paths leave {dest}: {', '.join(unsafe)}")
|
||||
if len(expected) != len(files):
|
||||
module.fail_json(msg="files repeat a path")
|
||||
outside = sorted(set(names) - set(params["tree"]))
|
||||
if outside:
|
||||
module.fail_json(msg=f"paths missing from tree: {', '.join(outside)}")
|
||||
refused = refusals(dest, files, settled, directories, params["tree"])
|
||||
if refused:
|
||||
module.exit_json(changed=False, refused=refused)
|
||||
|
||||
changed = False
|
||||
with tarfile.open(params["src"], mode="r:") as archive:
|
||||
found = members(module, archive, expected)
|
||||
mismatched = [
|
||||
name for name in files if sha1(archive, found[name]) != expected[name]
|
||||
]
|
||||
if mismatched:
|
||||
module.fail_json(
|
||||
msg=f"archived files do not match their checksum: {', '.join(mismatched)}"
|
||||
)
|
||||
for name in params["tree"]:
|
||||
target = dest / name
|
||||
if name in kept:
|
||||
changed = attributes(module, target, params["mode"], changed)
|
||||
if name not in expected:
|
||||
continue
|
||||
try:
|
||||
changed = (
|
||||
make_directories(module, target.parent, directory_mode) or changed
|
||||
)
|
||||
write(module, archive, found[name], target, module.tmpdir)
|
||||
except Exception as error: # noqa: BLE001 - atomic_move raises a bare Exception
|
||||
module.fail_json(
|
||||
msg=f"Failed to write {target}: {error}",
|
||||
path=str(target),
|
||||
changed=changed,
|
||||
)
|
||||
changed = attributes(module, target, params["mode"], True)
|
||||
for name in directories:
|
||||
path = dest / name
|
||||
try:
|
||||
created = make_directories(module, path, directory_mode)
|
||||
except OSError as error:
|
||||
module.fail_json(
|
||||
msg=f"Failed to create {path}: {error}",
|
||||
path=str(path),
|
||||
changed=changed,
|
||||
)
|
||||
changed = created or attributes(module, path, directory_mode, changed)
|
||||
module.exit_json(changed=changed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,11 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import tarfile
|
||||
import tempfile
|
||||
from pathlib import Path, PurePath
|
||||
from typing import Any
|
||||
|
||||
from ansible import constants
|
||||
from ansible.errors import AnsibleError
|
||||
from ansible.module_utils.parsing.convert_bool import boolean as _to_bool
|
||||
from ansible.plugins.action import copy as _copy_action
|
||||
@@ -29,6 +34,8 @@ _TREE_CHECK_ARGS = frozenset(
|
||||
}
|
||||
)
|
||||
_OCTAL_MODE = re.compile(r"^[0-7]{3,4}$")
|
||||
_ARCHIVE = ".source.tar"
|
||||
_ARCHIVE_LIMIT = 16 * 1024 * 1024
|
||||
|
||||
|
||||
def _octal(mode: Any) -> int | None:
|
||||
@@ -83,9 +90,9 @@ class ActionModule(CopyActionModule):
|
||||
}
|
||||
|
||||
self._task.args.setdefault("mode", "0644")
|
||||
unchanged = self._unchanged_tree(task_vars)
|
||||
if unchanged is not None:
|
||||
return unchanged
|
||||
settled = self._settled_tree(task_vars)
|
||||
if settled is not None:
|
||||
return settled
|
||||
walk = _copy_action._walk_dirs
|
||||
_copy_action._walk_dirs = _walk_dirs_without_bytecode_caches
|
||||
try:
|
||||
@@ -93,19 +100,22 @@ class ActionModule(CopyActionModule):
|
||||
finally:
|
||||
_copy_action._walk_dirs = walk
|
||||
|
||||
def _unchanged_tree(self, task_vars: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Settle a directory copy whose entries all match in one remote call.
|
||||
def _settled_tree(self, task_vars: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Settle a directory copy in one checksum call and one write call.
|
||||
|
||||
The copy action spends a stat and a file module call per file, and a
|
||||
file module call per directory holding no file, even when nothing
|
||||
differs; any difference, and any argument or source shape not
|
||||
modelled here, leaves the decision to the regular copy.
|
||||
The copy action spends a stat and a file module call per file, plus a
|
||||
transfer and a copy module call per differing file, and a file module
|
||||
call per directory holding no file. Here one remote call compares the
|
||||
whole tree; when entries differ outside check and diff mode, the
|
||||
differing files travel in one archive and one module call writes them.
|
||||
Any argument or source shape not modelled here leaves the decision to
|
||||
the regular copy.
|
||||
|
||||
Args:
|
||||
task_vars: Variables of the task, handed to the remote call.
|
||||
task_vars: Variables of the task, handed to the remote calls.
|
||||
|
||||
Returns:
|
||||
The unchanged copy result, or ``None`` when the regular copy runs.
|
||||
The copy result, or ``None`` when the regular copy runs.
|
||||
"""
|
||||
args = self._task.args
|
||||
src, dest, mode = args.get("src"), args.get("dest"), _octal(args.get("mode"))
|
||||
@@ -116,6 +126,7 @@ class ActionModule(CopyActionModule):
|
||||
or not src.endswith(os.path.sep)
|
||||
or not isinstance(dest, str)
|
||||
or not dest.startswith("/")
|
||||
or "$" in dest
|
||||
or mode is None
|
||||
or not _to_bool(args.get("force", True), strict=False)
|
||||
):
|
||||
@@ -146,7 +157,7 @@ class ActionModule(CopyActionModule):
|
||||
return None
|
||||
|
||||
dest = shell.join_path(dest, "")
|
||||
directories = [shell.join_path(dest, rel) for rel in leaves]
|
||||
directories = {shell.join_path(dest, rel): rel for rel in leaves}
|
||||
decrypt = _to_bool(args.get("decrypt", True), strict=False)
|
||||
wanted = {}
|
||||
for full, rel in tree["files"]:
|
||||
@@ -155,7 +166,7 @@ class ActionModule(CopyActionModule):
|
||||
except AnsibleError:
|
||||
return None
|
||||
try:
|
||||
wanted[shell.join_path(dest, rel)] = checksum(real)
|
||||
wanted[shell.join_path(dest, rel)] = (full, rel, checksum(real))
|
||||
finally:
|
||||
self._loader.cleanup_tmp_file(real)
|
||||
|
||||
@@ -167,21 +178,148 @@ class ActionModule(CopyActionModule):
|
||||
states = found.get("states")
|
||||
if found.get("failed") or not isinstance(states, dict):
|
||||
return None
|
||||
for path, digest in wanted.items():
|
||||
content, settled = {}, []
|
||||
for path, (full, rel, digest) in wanted.items():
|
||||
state = states.get(path) or {}
|
||||
if (
|
||||
not state.get("isreg")
|
||||
or state.get("checksum") != digest
|
||||
or _octal(state.get("mode")) != mode
|
||||
):
|
||||
return None
|
||||
for path in directories:
|
||||
if not state.get("isreg") or state.get("checksum") != digest:
|
||||
content[rel] = full
|
||||
elif _octal(state.get("mode")) != mode:
|
||||
settled.append(rel)
|
||||
drifted = []
|
||||
for path, rel in directories.items():
|
||||
state = states.get(path) or {}
|
||||
if not state.get("isdir") or (
|
||||
directory_mode is not None
|
||||
and _octal(state.get("mode")) != _octal(directory_mode)
|
||||
):
|
||||
return None
|
||||
drifted.append(rel)
|
||||
|
||||
if not (content or settled or drifted):
|
||||
self._remove_tmp_path(shell.tmpdir)
|
||||
return self._ensure_invocation(
|
||||
{"changed": False, "dest": dest, "src": source}
|
||||
)
|
||||
if self._task.check_mode or self._task.diff or self._is_become_unprivileged():
|
||||
return None
|
||||
return self._write_tree(
|
||||
task_vars,
|
||||
source=source,
|
||||
dest=dest,
|
||||
content=content,
|
||||
settled=settled,
|
||||
directories=drifted,
|
||||
tree=[*(rel for _, rel in tree["files"]), *leaves],
|
||||
decrypt=decrypt,
|
||||
)
|
||||
|
||||
def _write_tree(
|
||||
self,
|
||||
task_vars: dict[str, Any],
|
||||
*,
|
||||
source: str,
|
||||
dest: str,
|
||||
content: dict[str, str],
|
||||
settled: list[str],
|
||||
directories: list[str],
|
||||
tree: list[str],
|
||||
decrypt: bool,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Write the differing entries of a directory copy in one module call.
|
||||
|
||||
Args:
|
||||
task_vars: Variables of the task, handed to the remote call.
|
||||
source: The local source directory, with a trailing slash.
|
||||
dest: The remote destination directory, with a trailing slash.
|
||||
content: Relative path to local path of every file whose remote
|
||||
content differs or is missing, in walk order.
|
||||
settled: Relative paths of the files whose mode alone differs.
|
||||
directories: Relative paths of the leaf directories that are
|
||||
missing or whose mode differs.
|
||||
tree: Relative paths of every file and leaf directory of the
|
||||
source, for the remote check that no two reach one entry.
|
||||
decrypt: Whether vaulted source files are decrypted.
|
||||
|
||||
Returns:
|
||||
The copy result, or ``None`` when the archive would exceed
|
||||
``_ARCHIVE_LIMIT`` or the module refuses a remote entry, and the
|
||||
regular copy runs.
|
||||
"""
|
||||
shell = self._connection._shell
|
||||
descriptor, archive = tempfile.mkstemp(
|
||||
dir=constants.DEFAULT_LOCAL_TMP, prefix="."
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
files = self._pack(handle, content, decrypt)
|
||||
if files is None:
|
||||
return None
|
||||
if shell.tmpdir is None:
|
||||
self._make_tmp_path()
|
||||
remote = self._transfer_file(
|
||||
archive, shell.join_path(shell.tmpdir, _ARCHIVE)
|
||||
)
|
||||
finally:
|
||||
Path(archive).unlink()
|
||||
self._fixup_perms2((shell.tmpdir, remote))
|
||||
written = self._execute_module(
|
||||
module_name="ansible.legacy.copy_batch",
|
||||
module_args={
|
||||
"src": remote,
|
||||
"dest": dest,
|
||||
"files": files,
|
||||
"settled": settled,
|
||||
"directories": directories,
|
||||
"tree": tree,
|
||||
"mode": self._task.args["mode"],
|
||||
"directory_mode": self._task.args.get("directory_mode"),
|
||||
},
|
||||
task_vars=task_vars,
|
||||
)
|
||||
if written.get("refused"):
|
||||
return None
|
||||
self._remove_tmp_path(shell.tmpdir)
|
||||
return self._ensure_invocation({"changed": False, "dest": dest, "src": source})
|
||||
if written.get("failed"):
|
||||
return self._ensure_invocation(written)
|
||||
return self._ensure_invocation(
|
||||
{"changed": written["changed"], "dest": dest, "src": source}
|
||||
)
|
||||
|
||||
def _pack(
|
||||
self, handle: Any, content: dict[str, str], decrypt: bool
|
||||
) -> list[dict[str, str]] | None:
|
||||
"""Write the files of *content* as regular members of a tar archive.
|
||||
|
||||
Args:
|
||||
handle: Binary file object the archive is written to.
|
||||
content: Relative path to local path of every file, in walk order.
|
||||
decrypt: Whether vaulted source files are decrypted.
|
||||
|
||||
Returns:
|
||||
``{path, checksum}`` of every member in walk order, the checksum
|
||||
taken from the bytes packed, or ``None`` once the members exceed
|
||||
``_ARCHIVE_LIMIT`` bytes.
|
||||
"""
|
||||
files, size = [], 0
|
||||
with tarfile.open(fileobj=handle, mode="w") as bundle:
|
||||
for rel, full in content.items():
|
||||
real = self._loader.get_real_file(full, decrypt=decrypt)
|
||||
try:
|
||||
with Path(real).open("rb") as source:
|
||||
data = source.read(_ARCHIVE_LIMIT + 1 - size)
|
||||
finally:
|
||||
self._loader.cleanup_tmp_file(real)
|
||||
size += len(data)
|
||||
if size > _ARCHIVE_LIMIT:
|
||||
return None
|
||||
member = tarfile.TarInfo(rel)
|
||||
member.size = len(data)
|
||||
bundle.addfile(member, io.BytesIO(data))
|
||||
files.append(
|
||||
{
|
||||
"path": rel,
|
||||
"checksum": hashlib.sha1(
|
||||
data, usedforsecurity=False
|
||||
).hexdigest(),
|
||||
}
|
||||
)
|
||||
return files
|
||||
@@ -0,0 +1,714 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
from unittest.mock import patch
|
||||
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
from ansible.module_utils.testing import patch_module_args
|
||||
from ansible.modules import copy as copy_module
|
||||
from ansible.modules import file as file_module
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from types import ModuleType
|
||||
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
MODULE_PATH = PROJECT_ROOT / "library" / "copy_batch.py"
|
||||
|
||||
|
||||
def _load_copy_batch() -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"library_copy_batch", str(MODULE_PATH)
|
||||
)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError(f"Could not load: {MODULE_PATH}")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod) # type: ignore[attr-defined]
|
||||
return mod
|
||||
|
||||
|
||||
_MOD = _load_copy_batch()
|
||||
|
||||
|
||||
def _sha1(data):
|
||||
return hashlib.sha1(data, usedforsecurity=False).hexdigest()
|
||||
|
||||
|
||||
def _umask():
|
||||
current = os.umask(0)
|
||||
os.umask(current)
|
||||
return current
|
||||
|
||||
|
||||
def _call(main, args):
|
||||
buffer = io.StringIO()
|
||||
with (
|
||||
patch_module_args(args),
|
||||
contextlib.redirect_stdout(buffer),
|
||||
contextlib.suppress(SystemExit),
|
||||
):
|
||||
main()
|
||||
return json.loads(buffer.getvalue())
|
||||
|
||||
|
||||
class _Tree(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
self.dest = self.root / "dest"
|
||||
self.dest.mkdir()
|
||||
self.work = self.root / "work"
|
||||
self.work.mkdir()
|
||||
self.archive = self.root / "archive.tar"
|
||||
|
||||
def tearDown(self):
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _pack(self, files, *, names=None, kinds=None):
|
||||
with tarfile.open(self.archive, "w") as bundle:
|
||||
for name, data in files.items():
|
||||
member = tarfile.TarInfo((names or {}).get(name, name))
|
||||
member.type = (kinds or {}).get(name, tarfile.REGTYPE)
|
||||
member.size = len(data) if member.isreg() else 0
|
||||
bundle.addfile(member, io.BytesIO(data) if member.isreg() else None)
|
||||
return {name: _sha1(data) for name, data in files.items()}
|
||||
|
||||
def _run(
|
||||
self,
|
||||
files,
|
||||
*,
|
||||
settled=(),
|
||||
directories=(),
|
||||
mode="0644",
|
||||
directory_mode=None,
|
||||
digests=None,
|
||||
names=None,
|
||||
kinds=None,
|
||||
tree=None,
|
||||
):
|
||||
packed = self._pack(files, names=names, kinds=kinds)
|
||||
listed = packed if digests is None else digests
|
||||
return _call(
|
||||
_MOD.main,
|
||||
{
|
||||
"src": str(self.archive),
|
||||
"dest": f"{self.dest}/",
|
||||
"files": [
|
||||
{"path": name, "checksum": digest}
|
||||
for name, digest in listed.items()
|
||||
],
|
||||
"settled": list(settled),
|
||||
"directories": list(directories),
|
||||
"tree": [*listed, *settled, *directories] if tree is None else tree,
|
||||
"mode": mode,
|
||||
"directory_mode": directory_mode,
|
||||
"_ansible_tmpdir": str(self.work),
|
||||
},
|
||||
)
|
||||
|
||||
def _write(self, rel, data, mode):
|
||||
path = self.dest / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
path.chmod(mode)
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _mode(path):
|
||||
return stat.S_IMODE(path.lstat().st_mode)
|
||||
|
||||
@staticmethod
|
||||
def _bytes(path):
|
||||
return path.read_bytes() # nocheck: cache-read
|
||||
|
||||
|
||||
class TestWrites(_Tree):
|
||||
def test_replaces_and_creates_files_with_mode(self):
|
||||
self._write("icon.png", b"old", 0o600)
|
||||
|
||||
result = self._run({"icon.png": b"new", "nested/icon.svg": b"<svg/>"})
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertNotIn("refused", result)
|
||||
self.assertEqual(self._bytes(self.dest / "icon.png"), b"new")
|
||||
self.assertEqual(self._bytes(self.dest / "nested/icon.svg"), b"<svg/>")
|
||||
self.assertEqual(self._mode(self.dest / "icon.png"), 0o644)
|
||||
self.assertEqual(self._mode(self.dest / "nested/icon.svg"), 0o644)
|
||||
|
||||
def test_created_parents_receive_the_directory_mode(self):
|
||||
self._run({"a/b/icon.svg": b"<svg/>"}, directory_mode="0750")
|
||||
|
||||
self.assertEqual(self._mode(self.dest / "a"), 0o750)
|
||||
self.assertEqual(self._mode(self.dest / "a/b"), 0o750)
|
||||
|
||||
def test_existing_parent_keeps_its_mode(self):
|
||||
(self.dest / "a").mkdir()
|
||||
(self.dest / "a").chmod(0o701)
|
||||
|
||||
self._run({"a/b/icon.svg": b"<svg/>"}, directory_mode="0750")
|
||||
|
||||
self.assertEqual(self._mode(self.dest / "a"), 0o701)
|
||||
self.assertEqual(self._mode(self.dest / "a/b"), 0o750)
|
||||
|
||||
def test_unset_directory_mode_keeps_the_umask_mode(self):
|
||||
self._run({"a/icon.svg": b"<svg/>"}, directory_mode=None)
|
||||
|
||||
self.assertEqual(self._mode(self.dest / "a"), 0o777 & ~_umask())
|
||||
|
||||
def test_existing_leaf_directory_receives_the_directory_mode(self):
|
||||
(self.dest / "leaf").mkdir()
|
||||
(self.dest / "leaf").chmod(0o700)
|
||||
|
||||
result = self._run({}, directories=["leaf"], directory_mode="0750")
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertEqual(self._mode(self.dest / "leaf"), 0o750)
|
||||
|
||||
def test_missing_leaf_directory_is_created_with_its_parents(self):
|
||||
result = self._run({}, directories=["a/leaf"], directory_mode="0750")
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertEqual(self._mode(self.dest / "a"), 0o750)
|
||||
self.assertEqual(self._mode(self.dest / "a/leaf"), 0o750)
|
||||
|
||||
def test_settled_file_keeps_its_content_and_inode(self):
|
||||
path = self._write("icon.png", b"png", 0o600)
|
||||
inode = path.stat().st_ino
|
||||
|
||||
result = self._run({}, settled=["icon.png"])
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertEqual(path.stat().st_ino, inode)
|
||||
self.assertEqual(self._bytes(path), b"png")
|
||||
self.assertEqual(self._mode(path), 0o644)
|
||||
|
||||
def test_settled_file_with_its_mode_reports_no_change(self):
|
||||
self._write("icon.png", b"png", 0o644)
|
||||
|
||||
result = self._run({}, settled=["icon.png"])
|
||||
|
||||
self.assertFalse(result["changed"])
|
||||
|
||||
def test_replaced_file_gets_the_owner_of_the_file_it_replaces(self):
|
||||
path = self._write("icon.png", b"old", 0o600)
|
||||
owner = (path.stat().st_uid, path.stat().st_gid)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
AnsibleModule,
|
||||
"atomic_move",
|
||||
autospec=True,
|
||||
side_effect=AnsibleModule.atomic_move,
|
||||
) as move,
|
||||
patch("os.chown", wraps=os.chown) as chown,
|
||||
):
|
||||
self._run({"icon.png": b"new"})
|
||||
|
||||
self.assertEqual(move.call_args.args[2], str(path))
|
||||
self.assertNotIn("keep_dest_attrs", move.call_args.kwargs)
|
||||
self.assertIn(owner, [call.args[1:] for call in chown.call_args_list])
|
||||
self.assertEqual((path.stat().st_uid, path.stat().st_gid), owner)
|
||||
|
||||
def test_files_outside_the_call_stay(self):
|
||||
other = self._write("other.png", b"other", 0o600)
|
||||
|
||||
self._run({"icon.png": b"new"})
|
||||
|
||||
self.assertEqual(self._bytes(other), b"other")
|
||||
self.assertEqual(self._mode(other), 0o600)
|
||||
|
||||
def test_symlinked_parent_directory_is_written_through(self):
|
||||
real = self.root / "real"
|
||||
real.mkdir()
|
||||
(self.dest / "linked").symlink_to(real)
|
||||
|
||||
result = self._run({"linked/icon.svg": b"<svg/>"})
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertEqual(self._bytes(real / "icon.svg"), b"<svg/>")
|
||||
|
||||
def test_staged_copies_leave_no_trace(self):
|
||||
self._run({"icon.png": b"new", "nested/icon.svg": b"<svg/>"})
|
||||
|
||||
self.assertEqual(list(self.work.iterdir()), [])
|
||||
self.assertEqual(
|
||||
sorted(p.name for p in self.dest.iterdir()), ["icon.png", "nested"]
|
||||
)
|
||||
|
||||
|
||||
class TestRefusals(_Tree):
|
||||
def _assert_refused(self, result, path):
|
||||
self.assertFalse(result["changed"])
|
||||
self.assertTrue(any(str(path) in reason for reason in result["refused"]))
|
||||
self.assertFalse((self.dest / "fresh.png").exists())
|
||||
|
||||
def test_symlink_in_place_of_a_file_is_refused_and_kept(self):
|
||||
target = self.root / "target.png"
|
||||
target.write_bytes(b"target")
|
||||
(self.dest / "icon.png").symlink_to(target)
|
||||
|
||||
result = self._run({"icon.png": b"new", "fresh.png": b"fresh"})
|
||||
|
||||
self._assert_refused(result, self.dest / "icon.png")
|
||||
self.assertTrue((self.dest / "icon.png").is_symlink())
|
||||
self.assertEqual(self._bytes(target), b"target")
|
||||
|
||||
def test_dangling_symlink_in_place_of_a_file_is_refused(self):
|
||||
(self.dest / "icon.png").symlink_to(self.root / "absent")
|
||||
|
||||
result = self._run({"icon.png": b"new", "fresh.png": b"fresh"})
|
||||
|
||||
self._assert_refused(result, self.dest / "icon.png")
|
||||
|
||||
def test_directory_in_place_of_a_file_is_refused(self):
|
||||
(self.dest / "icon.png").mkdir()
|
||||
|
||||
result = self._run({"icon.png": b"new", "fresh.png": b"fresh"})
|
||||
|
||||
self._assert_refused(result, self.dest / "icon.png")
|
||||
self.assertTrue((self.dest / "icon.png").is_dir())
|
||||
|
||||
def test_fifo_in_place_of_a_file_is_refused(self):
|
||||
os.mkfifo(self.dest / "icon.png")
|
||||
|
||||
result = self._run({"icon.png": b"new", "fresh.png": b"fresh"})
|
||||
|
||||
self._assert_refused(result, self.dest / "icon.png")
|
||||
|
||||
def test_file_in_place_of_a_parent_directory_is_refused(self):
|
||||
self._write("nested", b"file", 0o644)
|
||||
|
||||
result = self._run({"nested/icon.svg": b"<svg/>", "fresh.png": b"fresh"})
|
||||
|
||||
self._assert_refused(result, self.dest / "nested")
|
||||
self.assertEqual(self._bytes(self.dest / "nested"), b"file")
|
||||
|
||||
def test_dangling_symlink_in_place_of_a_parent_directory_is_refused(self):
|
||||
(self.dest / "nested").symlink_to(self.root / "absent")
|
||||
|
||||
result = self._run({"nested/a/icon.svg": b"<svg/>", "fresh.png": b"fresh"})
|
||||
|
||||
self._assert_refused(result, self.dest / "nested")
|
||||
self.assertFalse((self.root / "absent").exists())
|
||||
|
||||
def test_symlink_in_place_of_a_leaf_directory_is_refused(self):
|
||||
(self.root / "real").mkdir()
|
||||
(self.dest / "leaf").symlink_to(self.root / "real")
|
||||
|
||||
result = self._run(
|
||||
{"fresh.png": b"fresh"}, directories=["leaf"], directory_mode="0750"
|
||||
)
|
||||
|
||||
self._assert_refused(result, self.dest / "leaf")
|
||||
self.assertNotEqual(self._mode(self.root / "real"), 0o750)
|
||||
|
||||
def test_file_in_place_of_a_leaf_directory_is_refused(self):
|
||||
self._write("leaf", b"", 0o644)
|
||||
|
||||
result = self._run({"fresh.png": b"fresh"}, directories=["leaf"])
|
||||
|
||||
self._assert_refused(result, self.dest / "leaf")
|
||||
|
||||
def test_symlink_loop_in_place_of_a_parent_directory_is_refused(self):
|
||||
(self.dest / "loop").symlink_to(self.dest / "loop")
|
||||
|
||||
result = self._run({"loop/icon.svg": b"<svg/>", "fresh.png": b"fresh"})
|
||||
|
||||
self._assert_refused(result, self.dest / "loop")
|
||||
|
||||
def test_missing_settled_file_is_refused(self):
|
||||
result = self._run({"fresh.png": b"fresh"}, settled=["icon.png"])
|
||||
|
||||
self._assert_refused(result, self.dest / "icon.png")
|
||||
|
||||
|
||||
class TestFailures(_Tree):
|
||||
def _assert_failed(self, result, needle):
|
||||
self.assertTrue(result["failed"])
|
||||
self.assertIn(needle, result["msg"])
|
||||
self.assertFalse((self.dest / "icon.png").exists())
|
||||
|
||||
def test_checksum_mismatch_fails_naming_the_file_and_writes_nothing(self):
|
||||
digests = {"icon.png": _sha1(b"new"), "nested/icon.svg": _sha1(b"other")}
|
||||
|
||||
result = self._run(
|
||||
{"icon.png": b"new", "nested/icon.svg": b"<svg/>"}, digests=digests
|
||||
)
|
||||
|
||||
self._assert_failed(result, "nested/icon.svg")
|
||||
self.assertFalse((self.dest / "nested").exists())
|
||||
|
||||
def test_parent_reference_member_fails(self):
|
||||
result = self._run(
|
||||
{"icon.png": b"new", "evil": b"x"},
|
||||
names={"evil": "../evil"},
|
||||
digests={"icon.png": _sha1(b"new")},
|
||||
)
|
||||
|
||||
self._assert_failed(result, "../evil")
|
||||
self.assertFalse((self.root / "evil").exists())
|
||||
|
||||
def test_absolute_member_fails(self):
|
||||
result = self._run(
|
||||
{"icon.png": b"new", "evil": b"x"},
|
||||
names={"evil": f"{self.root}/evil"},
|
||||
digests={"icon.png": _sha1(b"new")},
|
||||
)
|
||||
|
||||
self._assert_failed(result, "leaves the destination")
|
||||
self.assertFalse((self.root / "evil").exists())
|
||||
|
||||
def test_unexpected_member_fails(self):
|
||||
result = self._run(
|
||||
{"icon.png": b"new", "extra.png": b"x"},
|
||||
digests={"icon.png": _sha1(b"new")},
|
||||
)
|
||||
|
||||
self._assert_failed(result, "extra.png")
|
||||
|
||||
def test_missing_member_fails(self):
|
||||
result = self._run(
|
||||
{"icon.png": b"new"},
|
||||
digests={"icon.png": _sha1(b"new"), "gone.png": _sha1(b"x")},
|
||||
)
|
||||
|
||||
self._assert_failed(result, "gone.png")
|
||||
|
||||
def test_link_member_fails(self):
|
||||
result = self._run(
|
||||
{"icon.png": b"new", "link.png": b""},
|
||||
kinds={"link.png": tarfile.SYMTYPE},
|
||||
)
|
||||
|
||||
self._assert_failed(result, "not a regular file")
|
||||
|
||||
def test_parent_reference_in_the_arguments_fails(self):
|
||||
result = self._run({}, directories=["../outside"])
|
||||
|
||||
self.assertTrue(result["failed"])
|
||||
self.assertIn("../outside", result["msg"])
|
||||
self.assertFalse((self.root / "outside").exists())
|
||||
|
||||
def test_write_failure_names_the_file_and_fails(self):
|
||||
original = AnsibleModule.atomic_move
|
||||
|
||||
def move(module, src, dest, **kwargs):
|
||||
if dest.endswith("nested/icon.svg"):
|
||||
raise Exception("Could not replace") # noqa: TRY002 - atomic_move raises a bare Exception
|
||||
return original(module, src, dest, **kwargs)
|
||||
|
||||
with patch.object(
|
||||
AnsibleModule, "atomic_move", autospec=True, side_effect=move
|
||||
):
|
||||
result = self._run({"icon.png": b"new", "nested/icon.svg": b"<svg/>"})
|
||||
|
||||
self.assertTrue(result["failed"])
|
||||
self.assertIn(str(self.dest / "nested/icon.svg"), result["msg"])
|
||||
self.assertTrue(result["changed"])
|
||||
|
||||
def test_failure_after_creating_a_parent_reports_the_change(self):
|
||||
with patch.object(
|
||||
AnsibleModule,
|
||||
"atomic_move",
|
||||
autospec=True,
|
||||
side_effect=Exception("Could not replace"),
|
||||
):
|
||||
result = self._run({"nested/icon.svg": b"<svg/>"})
|
||||
|
||||
self.assertTrue(result["failed"])
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertTrue((self.dest / "nested").is_dir())
|
||||
|
||||
def test_repeated_file_path_fails(self):
|
||||
result = self._run(
|
||||
{"fresh.png": b"fresh"},
|
||||
digests={"fresh.png": _sha1(b"fresh")},
|
||||
tree=["fresh.png"],
|
||||
)
|
||||
self.assertNotIn("failed", result)
|
||||
|
||||
repeated = _call(
|
||||
_MOD.main,
|
||||
{
|
||||
"src": str(self.archive),
|
||||
"dest": f"{self.dest}/",
|
||||
"files": [
|
||||
{"path": "fresh.png", "checksum": _sha1(b"fresh")},
|
||||
{"path": "fresh.png", "checksum": _sha1(b"fresh")},
|
||||
],
|
||||
"settled": [],
|
||||
"directories": [],
|
||||
"tree": ["fresh.png"],
|
||||
"mode": "0644",
|
||||
"directory_mode": None,
|
||||
"_ansible_tmpdir": str(self.work),
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(repeated["failed"])
|
||||
self.assertIn("repeat", repeated["msg"])
|
||||
|
||||
|
||||
class TestOrderAndAliases(_Tree):
|
||||
def test_files_are_written_in_the_given_order(self):
|
||||
order = ["b/z.png", "a.png", "c.png"]
|
||||
|
||||
with patch.object(
|
||||
AnsibleModule,
|
||||
"atomic_move",
|
||||
autospec=True,
|
||||
side_effect=AnsibleModule.atomic_move,
|
||||
) as move:
|
||||
self._run({name: name.encode() for name in order})
|
||||
|
||||
self.assertEqual(
|
||||
[call.args[2] for call in move.call_args_list],
|
||||
[str(self.dest / name) for name in order],
|
||||
)
|
||||
|
||||
def test_settled_files_are_handled_in_tree_order(self):
|
||||
settled = self._write("s.png", b"s", 0o600)
|
||||
|
||||
with patch.object(
|
||||
AnsibleModule,
|
||||
"atomic_move",
|
||||
autospec=True,
|
||||
side_effect=Exception("Could not replace"),
|
||||
):
|
||||
result = self._run(
|
||||
{"b.png": b"b"}, settled=["s.png"], tree=["s.png", "b.png"]
|
||||
)
|
||||
|
||||
self.assertTrue(result["failed"])
|
||||
self.assertEqual(self._mode(settled), 0o644)
|
||||
|
||||
def test_settled_file_after_a_failed_write_stays_untouched(self):
|
||||
settled = self._write("s.png", b"s", 0o600)
|
||||
|
||||
with patch.object(
|
||||
AnsibleModule,
|
||||
"atomic_move",
|
||||
autospec=True,
|
||||
side_effect=Exception("Could not replace"),
|
||||
):
|
||||
result = self._run(
|
||||
{"b.png": b"b"}, settled=["s.png"], tree=["b.png", "s.png"]
|
||||
)
|
||||
|
||||
self.assertTrue(result["failed"])
|
||||
self.assertEqual(self._mode(settled), 0o600)
|
||||
|
||||
def test_path_missing_from_tree_fails(self):
|
||||
result = self._run({"b.png": b"b"}, tree=[])
|
||||
|
||||
self.assertTrue(result["failed"])
|
||||
self.assertIn("missing from tree", result["msg"])
|
||||
|
||||
def test_two_paths_through_a_symlinked_directory_are_refused(self):
|
||||
(self.dest / "y").mkdir()
|
||||
self._write("y/f.js", b"y", 0o644)
|
||||
(self.dest / "x").symlink_to(self.dest / "y")
|
||||
|
||||
result = self._run(
|
||||
{"x/f.js": b"x", "fresh.png": b"fresh"},
|
||||
tree=["x/f.js", "y/f.js", "fresh.png"],
|
||||
)
|
||||
|
||||
self.assertFalse(result["changed"])
|
||||
self.assertIn("reach the same entry", " ".join(result["refused"]))
|
||||
self.assertEqual(self._bytes(self.dest / "y/f.js"), b"y")
|
||||
self.assertFalse((self.dest / "fresh.png").exists())
|
||||
|
||||
def test_two_missing_paths_through_a_symlinked_directory_are_refused(self):
|
||||
(self.dest / "y").mkdir()
|
||||
(self.dest / "x").symlink_to(self.dest / "y")
|
||||
|
||||
result = self._run({"x/f.js": b"x", "y/f.js": b"y"}, tree=["x/f.js", "y/f.js"])
|
||||
|
||||
self.assertIn("reach the same entry", " ".join(result["refused"]))
|
||||
self.assertFalse((self.dest / "y/f.js").exists())
|
||||
|
||||
def test_hard_linked_targets_are_refused(self):
|
||||
first = self._write("a.png", b"same", 0o644)
|
||||
(self.dest / "b.png").hardlink_to(first)
|
||||
|
||||
result = self._run({"a.png": b"new"}, tree=["a.png", "b.png"])
|
||||
|
||||
self.assertIn("reach the same entry", " ".join(result["refused"]))
|
||||
self.assertEqual(self._bytes(self.dest / "b.png"), b"same")
|
||||
|
||||
def test_unsafe_writes_reaches_atomic_move_from_the_environment(self):
|
||||
with (
|
||||
patch.dict(os.environ, {"ANSIBLE_UNSAFE_WRITES": "1"}),
|
||||
patch.object(
|
||||
AnsibleModule,
|
||||
"atomic_move",
|
||||
autospec=True,
|
||||
side_effect=AnsibleModule.atomic_move,
|
||||
) as move,
|
||||
):
|
||||
self._run({"icon.png": b"new"})
|
||||
|
||||
self.assertTrue(move.call_args.kwargs["unsafe_writes"])
|
||||
|
||||
def test_leaf_created_concurrently_is_accepted(self):
|
||||
original = Path.mkdir
|
||||
|
||||
def racing(path, *args, **kwargs):
|
||||
original(path, *args, **kwargs)
|
||||
raise FileExistsError(path)
|
||||
|
||||
with patch.object(Path, "mkdir", autospec=True, side_effect=racing):
|
||||
result = self._run({}, directories=["leaf"], directory_mode="0750")
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertEqual(self._mode(self.dest / "leaf"), 0o750)
|
||||
|
||||
|
||||
class TestSameEndStateAsTheCopyModule(unittest.TestCase):
|
||||
MODE = "0644"
|
||||
DIRECTORY_MODE = "0750"
|
||||
CONTENT: ClassVar[dict[str, bytes]] = {
|
||||
"replaced.png": b"new",
|
||||
"fresh/deep/new.svg": b"<svg/>",
|
||||
"kept-dir/new.png": b"png",
|
||||
}
|
||||
SETTLED: ClassVar[tuple[str, ...]] = ("mode-drift.png",)
|
||||
LEAVES: ClassVar[tuple[str, ...]] = ("existing-leaf", "absent/leaf")
|
||||
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _seed(self, dest):
|
||||
dest.mkdir()
|
||||
(dest / "replaced.png").write_bytes(b"old")
|
||||
(dest / "replaced.png").chmod(0o600)
|
||||
(dest / "mode-drift.png").write_bytes(b"same")
|
||||
(dest / "mode-drift.png").chmod(0o600)
|
||||
(dest / "kept-dir").mkdir()
|
||||
(dest / "kept-dir").chmod(0o701)
|
||||
(dest / "existing-leaf").mkdir()
|
||||
(dest / "existing-leaf").chmod(0o700)
|
||||
(dest / "untouched.png").write_bytes(b"untouched")
|
||||
|
||||
def _regular_copy(self, dest):
|
||||
staging = self.root / "staging"
|
||||
staging.mkdir()
|
||||
for rel, data in self.CONTENT.items():
|
||||
source = staging / ".source"
|
||||
source.write_bytes(data)
|
||||
result = _call(
|
||||
copy_module.main,
|
||||
{
|
||||
"src": str(source),
|
||||
"dest": f"{dest}/",
|
||||
"mode": self.MODE,
|
||||
"directory_mode": self.DIRECTORY_MODE,
|
||||
"_original_basename": rel,
|
||||
"follow": False,
|
||||
"checksum": _sha1(data),
|
||||
},
|
||||
)
|
||||
self.assertNotIn("failed", result, result)
|
||||
for rel in self.SETTLED:
|
||||
result = _call(
|
||||
file_module.main,
|
||||
{
|
||||
"mode": self.MODE,
|
||||
"dest": f"{dest}/",
|
||||
"_original_basename": rel,
|
||||
"recurse": False,
|
||||
"state": "file",
|
||||
},
|
||||
)
|
||||
self.assertNotIn("failed", result, result)
|
||||
for rel in self.LEAVES:
|
||||
result = _call(
|
||||
file_module.main,
|
||||
{
|
||||
"path": str(dest / rel),
|
||||
"state": "directory",
|
||||
"mode": self.DIRECTORY_MODE,
|
||||
"recurse": False,
|
||||
},
|
||||
)
|
||||
self.assertNotIn("failed", result, result)
|
||||
|
||||
def _batch(self, dest):
|
||||
archive = self.root / "archive.tar"
|
||||
work = self.root / "work"
|
||||
work.mkdir()
|
||||
with tarfile.open(archive, "w") as bundle:
|
||||
for rel, data in self.CONTENT.items():
|
||||
member = tarfile.TarInfo(rel)
|
||||
member.size = len(data)
|
||||
bundle.addfile(member, io.BytesIO(data))
|
||||
result = _call(
|
||||
_MOD.main,
|
||||
{
|
||||
"src": str(archive),
|
||||
"dest": f"{dest}/",
|
||||
"files": [
|
||||
{"path": rel, "checksum": _sha1(data)}
|
||||
for rel, data in self.CONTENT.items()
|
||||
],
|
||||
"settled": list(self.SETTLED),
|
||||
"directories": list(self.LEAVES),
|
||||
"tree": [*self.CONTENT, *self.SETTLED, *self.LEAVES],
|
||||
"mode": self.MODE,
|
||||
"directory_mode": self.DIRECTORY_MODE,
|
||||
"_ansible_tmpdir": str(work),
|
||||
},
|
||||
)
|
||||
self.assertTrue(result["changed"], result)
|
||||
|
||||
@staticmethod
|
||||
def _snapshot(dest):
|
||||
entries = {}
|
||||
for path in sorted(dest.rglob("*")): # nocheck: project-walk
|
||||
info = path.lstat()
|
||||
content = None
|
||||
if path.is_file():
|
||||
content = path.read_bytes() # nocheck: cache-read
|
||||
entries[str(path.relative_to(dest))] = (
|
||||
stat.S_IFMT(info.st_mode),
|
||||
stat.S_IMODE(info.st_mode),
|
||||
info.st_uid,
|
||||
info.st_gid,
|
||||
content,
|
||||
)
|
||||
return entries
|
||||
|
||||
def test_batch_leaves_the_tree_the_regular_copy_leaves(self):
|
||||
regular, batch = self.root / "regular", self.root / "batch"
|
||||
self._seed(regular)
|
||||
self._seed(batch)
|
||||
|
||||
self._regular_copy(regular)
|
||||
self._batch(batch)
|
||||
|
||||
self.assertEqual(self._snapshot(batch), self._snapshot(regular))
|
||||
self.assertEqual(self._snapshot(batch)["kept-dir"][1], 0o701)
|
||||
self.assertEqual(self._snapshot(batch)["existing-leaf"][1], 0o750)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,5 +1,6 @@
|
||||
import hashlib
|
||||
import os
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
@@ -153,6 +154,7 @@ class _FakeShell:
|
||||
class _FakeConnection:
|
||||
def __init__(self):
|
||||
self._shell = _FakeShell()
|
||||
self.become = None
|
||||
|
||||
|
||||
class _FakeLoader:
|
||||
@@ -179,6 +181,11 @@ class _Dir:
|
||||
self.mode = mode
|
||||
|
||||
|
||||
class _Link:
|
||||
def __init__(self, data):
|
||||
self.data = data
|
||||
|
||||
|
||||
def _mode_bits(mode):
|
||||
return mode if isinstance(mode, int) else int(mode, 8)
|
||||
|
||||
@@ -188,6 +195,24 @@ class _FakeRemote:
|
||||
self.entries = dict(entries)
|
||||
self.calls = []
|
||||
self.args = []
|
||||
self.transfers = []
|
||||
self.fixups = 0
|
||||
|
||||
def transfer(self, local, remote_path):
|
||||
if not remote_path.endswith(".tar"):
|
||||
self.transfers.append(Path(local).read_bytes()) # nocheck: cache-read
|
||||
return remote_path
|
||||
with tarfile.open(local) as bundle:
|
||||
self.transfers.append(
|
||||
{
|
||||
member.name: bundle.extractfile(member).read()
|
||||
for member in bundle.getmembers()
|
||||
}
|
||||
)
|
||||
return remote_path
|
||||
|
||||
def fixup(self, *args, **kwargs):
|
||||
self.fixups += 1
|
||||
|
||||
def __call__(self, module_name=None, module_args=None, task_vars=None, **kwargs):
|
||||
self.calls.append(module_name)
|
||||
@@ -197,14 +222,31 @@ class _FakeRemote:
|
||||
"changed": False,
|
||||
"states": {p: self._state(p) for p in module_args["paths"]},
|
||||
}
|
||||
if module_name == "ansible.legacy.copy_batch":
|
||||
return self._copy_batch(module_args)
|
||||
if module_name == "ansible.legacy.stat":
|
||||
entry = self.entries.get(module_args["path"])
|
||||
if entry is None:
|
||||
return {"stat": {"exists": False}}
|
||||
if isinstance(entry, _Dir):
|
||||
return {"stat": {"exists": True, "isdir": True}}
|
||||
if isinstance(entry, _Link):
|
||||
digest = _sha1(entry.data) if module_args["follow"] else ""
|
||||
return {
|
||||
"stat": {
|
||||
"exists": True,
|
||||
"isdir": False,
|
||||
"islnk": True,
|
||||
"checksum": digest,
|
||||
}
|
||||
}
|
||||
return {
|
||||
"stat": {"exists": True, "isdir": False, "checksum": _sha1(entry[0])}
|
||||
"stat": {
|
||||
"exists": True,
|
||||
"isdir": False,
|
||||
"islnk": False,
|
||||
"checksum": _sha1(entry[0]),
|
||||
}
|
||||
}
|
||||
if module_name == "ansible.legacy.file" and module_args["state"] == "directory":
|
||||
return self._ensure_directory(module_args["path"], module_args["mode"])
|
||||
@@ -228,9 +270,39 @@ class _FakeRemote:
|
||||
"path": path,
|
||||
}
|
||||
|
||||
def _copy_batch(self, args):
|
||||
dest = args["dest"]
|
||||
files = [entry["path"] for entry in args["files"]]
|
||||
refused = [
|
||||
dest + rel
|
||||
for rel in [*files, *args["settled"]]
|
||||
if isinstance(self.entries.get(dest + rel), (_Dir, _Link))
|
||||
] + [
|
||||
dest + rel
|
||||
for rel in args["directories"]
|
||||
if dest + rel in self.entries
|
||||
and not isinstance(self.entries[dest + rel], _Dir)
|
||||
]
|
||||
if refused:
|
||||
return {"changed": False, "refused": refused}
|
||||
archive = self.transfers[-1]
|
||||
if list(archive) != files or any(
|
||||
_sha1(archive[entry["path"]]) != entry["checksum"]
|
||||
for entry in args["files"]
|
||||
):
|
||||
return {"failed": True, "msg": "archive does not match files"}
|
||||
mode = _mode_bits(args["mode"])
|
||||
for rel in files:
|
||||
self.entries[dest + rel] = (archive[rel], mode)
|
||||
for rel in args["settled"]:
|
||||
self.entries[dest + rel] = (self.entries[dest + rel][0], mode)
|
||||
for rel in args["directories"]:
|
||||
self.entries[dest + rel] = _Dir(_mode_bits(args["directory_mode"] or 0o755))
|
||||
return {"changed": True}
|
||||
|
||||
def _state(self, path):
|
||||
entry = self.entries.get(path)
|
||||
if entry is None:
|
||||
if entry is None or isinstance(entry, _Link):
|
||||
return {"isreg": False, "isdir": False, "mode": None, "checksum": None}
|
||||
if isinstance(entry, _Dir):
|
||||
return {
|
||||
@@ -247,12 +319,8 @@ class _FakeRemote:
|
||||
}
|
||||
|
||||
|
||||
class TestStackHostCopyTree(unittest.TestCase):
|
||||
FILES: ClassVar[dict[str, bytes]] = {
|
||||
"favicon.ico": b"ico",
|
||||
"icon.png": b"png",
|
||||
"nested/icon.svg": b"<svg/>",
|
||||
}
|
||||
class _TreeHarness(unittest.TestCase):
|
||||
FILES: ClassVar[dict[str, bytes]] = {}
|
||||
DEST = "/var/www/public_cdn/roles/app/latest/img/design/"
|
||||
|
||||
def setUp(self):
|
||||
@@ -268,15 +336,19 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
def _synced(self, mode=0o644):
|
||||
return {self.DEST + rel: (data, mode) for rel, data in self.FILES.items()}
|
||||
|
||||
def _run(self, remote, **extra):
|
||||
def _run(self, remote, *, check_mode=False, diff=False, loader=None, **extra):
|
||||
task = _FakeTask(args={"src": f"{self.src}/", "dest": self.DEST, **extra})
|
||||
task.check_mode = check_mode
|
||||
task.diff = diff
|
||||
action = _make_action(task, _FakeTemplar(is_stack_host=True))
|
||||
action._connection = _FakeConnection()
|
||||
action._loader = _FakeLoader()
|
||||
action._loader = _FakeLoader() if loader is None else loader
|
||||
action._play_context = _FakePlayContext()
|
||||
action._execute_module = remote
|
||||
action._transfer_file = lambda local, remote_path: remote_path
|
||||
action._fixup_perms2 = lambda *a, **k: None
|
||||
action._transfer_file = getattr(
|
||||
remote, "transfer", lambda local, remote_path: remote_path
|
||||
)
|
||||
action._fixup_perms2 = getattr(remote, "fixup", lambda *a, **k: None)
|
||||
action._remove_tmp_path = lambda *a, **k: None
|
||||
action._remote_expand_user = lambda path, **k: path
|
||||
|
||||
@@ -287,6 +359,36 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
action._make_tmp_path = make_tmp_path
|
||||
return action.run(task_vars={})
|
||||
|
||||
BATCH: ClassVar[list[str]] = [
|
||||
"ansible.legacy.file_checksums",
|
||||
"ansible.legacy.copy_batch",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _batch_args(remote):
|
||||
return remote.args[remote.calls.index("ansible.legacy.copy_batch")]
|
||||
|
||||
@staticmethod
|
||||
def _copies(remote):
|
||||
return [
|
||||
args
|
||||
for name, args in zip(remote.calls, remote.args, strict=True)
|
||||
if name == "ansible.legacy.copy"
|
||||
]
|
||||
|
||||
|
||||
class TestStackHostCopyTree(_TreeHarness):
|
||||
FILES: ClassVar[dict[str, bytes]] = {
|
||||
"favicon.ico": b"ico",
|
||||
"icon.png": b"png",
|
||||
"nested/icon.svg": b"<svg/>",
|
||||
}
|
||||
|
||||
def _changed(self):
|
||||
entries = self._synced()
|
||||
entries[self.DEST + "icon.png"] = (b"old", 0o644)
|
||||
return entries
|
||||
|
||||
def test_unchanged_tree_costs_one_remote_call(self):
|
||||
remote = _FakeRemote(self._synced())
|
||||
|
||||
@@ -297,7 +399,7 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
self.assertEqual(result["dest"], self.DEST)
|
||||
self.assertEqual(result["src"], f"{self.src}/")
|
||||
|
||||
def test_changed_file_runs_the_regular_copy(self):
|
||||
def test_changed_file_travels_alone_in_one_batch(self):
|
||||
entries = self._synced()
|
||||
entries[self.DEST + "icon.png"] = (b"old", 0o644)
|
||||
remote = _FakeRemote(entries)
|
||||
@@ -305,9 +407,16 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
result = self._run(remote)
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertIn("ansible.legacy.copy", remote.calls)
|
||||
self.assertEqual(remote.calls, self.BATCH)
|
||||
self.assertEqual(remote.transfers, [{"icon.png": b"png"}])
|
||||
self.assertEqual(
|
||||
self._batch_args(remote)["files"],
|
||||
[{"path": "icon.png", "checksum": _sha1(b"png")}],
|
||||
)
|
||||
self.assertEqual(sorted(self._batch_args(remote)["tree"]), sorted(self.FILES))
|
||||
self.assertEqual(remote.entries, self._synced())
|
||||
|
||||
def test_missing_file_runs_the_regular_copy(self):
|
||||
def test_missing_file_travels_in_one_batch(self):
|
||||
entries = self._synced()
|
||||
del entries[self.DEST + "nested/icon.svg"]
|
||||
remote = _FakeRemote(entries)
|
||||
@@ -315,9 +424,10 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
result = self._run(remote)
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertIn("ansible.legacy.copy", remote.calls)
|
||||
self.assertEqual(remote.calls, self.BATCH)
|
||||
self.assertEqual(remote.transfers, [{"nested/icon.svg": b"<svg/>"}])
|
||||
|
||||
def test_mode_drift_runs_the_regular_copy(self):
|
||||
def test_mode_drift_is_settled_without_sending_the_file(self):
|
||||
entries = self._synced()
|
||||
entries[self.DEST + "favicon.ico"] = (b"ico", 0o600)
|
||||
remote = _FakeRemote(entries)
|
||||
@@ -325,16 +435,165 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
result = self._run(remote)
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertIn("ansible.legacy.file", remote.calls)
|
||||
self.assertEqual(remote.calls, self.BATCH)
|
||||
self.assertEqual(remote.transfers, [{}])
|
||||
self.assertEqual(self._batch_args(remote)["settled"], ["favicon.ico"])
|
||||
self.assertEqual(self._batch_args(remote)["files"], [])
|
||||
|
||||
def test_directory_in_place_of_a_file_runs_the_regular_copy(self):
|
||||
def test_directory_in_place_of_a_file_is_refused_then_copied_regularly(self):
|
||||
entries = self._synced()
|
||||
entries[self.DEST + "icon.png"] = _Dir(0o755)
|
||||
remote = _FakeRemote(entries)
|
||||
|
||||
self._run(remote)
|
||||
|
||||
self.assertEqual(remote.calls[:2], self.BATCH)
|
||||
self.assertIn("ansible.legacy.stat", remote.calls)
|
||||
self.assertIn("ansible.legacy.copy", remote.calls)
|
||||
|
||||
def test_symlink_in_place_of_a_file_is_refused_then_copied_regularly(self):
|
||||
entries = self._synced()
|
||||
entries[self.DEST + "icon.png"] = _Link(b"png")
|
||||
remote = _FakeRemote(entries)
|
||||
|
||||
self._run(remote)
|
||||
|
||||
self.assertEqual(remote.calls[:2], self.BATCH)
|
||||
copies = self._copies(remote)
|
||||
self.assertEqual([args["_original_basename"] for args in copies], ["icon.png"])
|
||||
self.assertFalse(copies[0]["follow"])
|
||||
|
||||
def test_check_mode_keeps_the_regular_copy(self):
|
||||
remote = _FakeRemote(self._changed())
|
||||
|
||||
result = self._run(remote, check_mode=True)
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertNotIn("ansible.legacy.copy_batch", remote.calls)
|
||||
self.assertNotIn("ansible.legacy.copy", remote.calls)
|
||||
self.assertIn("ansible.legacy.stat", remote.calls)
|
||||
self.assertEqual(remote.transfers, [])
|
||||
|
||||
def test_diff_mode_keeps_the_regular_copy(self):
|
||||
remote = _FakeRemote(self._changed())
|
||||
|
||||
with patch(
|
||||
"plugins.action.stack_host_copy.CopyActionModule.run",
|
||||
return_value={"changed": True, "diff": [{"after": "png"}]},
|
||||
) as super_run:
|
||||
result = self._run(remote, diff=True)
|
||||
|
||||
super_run.assert_called_once()
|
||||
self.assertEqual(result["diff"], [{"after": "png"}])
|
||||
self.assertEqual(remote.calls, ["ansible.legacy.file_checksums"])
|
||||
self.assertEqual(remote.transfers, [])
|
||||
|
||||
def test_failed_batch_stays_failed_without_a_second_copy(self):
|
||||
failure = {
|
||||
"failed": True,
|
||||
"changed": True,
|
||||
"msg": f"Failed to write {self.DEST}icon.png: boom",
|
||||
}
|
||||
|
||||
class _FailingBatch(_FakeRemote):
|
||||
def _copy_batch(self, args):
|
||||
return dict(failure)
|
||||
|
||||
remote = _FailingBatch(self._changed())
|
||||
|
||||
with patch("plugins.action.stack_host_copy.CopyActionModule.run") as super_run:
|
||||
result = self._run(remote)
|
||||
|
||||
super_run.assert_not_called()
|
||||
self.assertTrue(result["failed"])
|
||||
self.assertIn(f"{self.DEST}icon.png", result["msg"])
|
||||
self.assertEqual(remote.calls, self.BATCH)
|
||||
|
||||
def test_batch_result_has_the_keys_of_the_regular_copy(self):
|
||||
batch = self._run(_FakeRemote(self._changed()))
|
||||
with patch.object(ActionModule, "_write_tree", return_value=None):
|
||||
regular = self._run(_FakeRemote(self._changed()))
|
||||
|
||||
self.assertEqual(set(batch), set(regular))
|
||||
self.assertEqual(
|
||||
{key: batch[key] for key in ("changed", "dest", "src")},
|
||||
{key: regular[key] for key in ("changed", "dest", "src")},
|
||||
)
|
||||
|
||||
def test_unprivileged_become_keeps_the_regular_copy(self):
|
||||
remote = _FakeRemote(self._changed())
|
||||
|
||||
with patch.object(ActionModule, "_is_become_unprivileged", return_value=True):
|
||||
result = self._run(remote)
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertNotIn("ansible.legacy.copy_batch", remote.calls)
|
||||
self.assertIn("ansible.legacy.copy", remote.calls)
|
||||
|
||||
def test_dest_with_a_variable_skips_the_tree_check(self):
|
||||
remote = _FakeRemote(self._changed())
|
||||
task = _FakeTask(args={"src": f"{self.src}/", "dest": "/srv/$APP/img/"})
|
||||
action = _make_action(task, _FakeTemplar(is_stack_host=True))
|
||||
action._execute_module = remote
|
||||
|
||||
with patch(
|
||||
"plugins.action.stack_host_copy.CopyActionModule.run",
|
||||
return_value={"changed": True},
|
||||
) as super_run:
|
||||
action.run(task_vars={})
|
||||
|
||||
super_run.assert_called_once()
|
||||
self.assertEqual(remote.calls, [])
|
||||
|
||||
def test_archive_over_the_limit_keeps_the_regular_copy(self):
|
||||
entries = self._synced()
|
||||
entries[self.DEST + "icon.png"] = (b"old", 0o644)
|
||||
entries[self.DEST + "favicon.ico"] = (b"old", 0o644)
|
||||
remote = _FakeRemote(entries)
|
||||
|
||||
with patch("plugins.action.stack_host_copy._ARCHIVE_LIMIT", 5):
|
||||
result = self._run(remote)
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertNotIn("ansible.legacy.copy_batch", remote.calls)
|
||||
self.assertEqual(sorted(remote.transfers), [b"ico", b"png"])
|
||||
|
||||
def test_files_travel_in_the_order_the_copy_action_walks(self):
|
||||
remote = _FakeRemote({})
|
||||
|
||||
self._run(remote)
|
||||
|
||||
walked = [
|
||||
rel
|
||||
for _, rel in _walk_dirs_without_bytecode_caches(
|
||||
f"{self.src}/", local_follow=True
|
||||
)["files"]
|
||||
]
|
||||
sent = [entry["path"] for entry in self._batch_args(remote)["files"]]
|
||||
self.assertEqual(sent, walked)
|
||||
self.assertEqual(list(remote.transfers[0]), walked)
|
||||
|
||||
def test_checksum_is_taken_from_the_packed_bytes(self):
|
||||
remote = _FakeRemote(self._changed())
|
||||
icon = self.src / "icon.png"
|
||||
loader = _FakeLoader()
|
||||
reads = []
|
||||
|
||||
def get_real_file(path, decrypt=True):
|
||||
reads.append(path)
|
||||
if path == str(icon) and reads.count(path) == 2:
|
||||
icon.write_bytes(b"edited")
|
||||
return path
|
||||
|
||||
loader.get_real_file = get_real_file
|
||||
|
||||
self._run(remote, loader=loader)
|
||||
|
||||
self.assertEqual(remote.transfers, [{"icon.png": b"edited"}])
|
||||
self.assertEqual(
|
||||
self._batch_args(remote)["files"],
|
||||
[{"path": "icon.png", "checksum": _sha1(b"edited")}],
|
||||
)
|
||||
|
||||
def test_requested_mode_is_what_the_tree_check_compares(self):
|
||||
remote = _FakeRemote(self._synced(mode=0o600))
|
||||
@@ -437,7 +696,7 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
regular = _FakeRemote(self._with_leaves(0o700))
|
||||
|
||||
settled = self._run(remote)
|
||||
with patch.object(ActionModule, "_unchanged_tree", return_value=None):
|
||||
with patch.object(ActionModule, "_settled_tree", return_value=None):
|
||||
copied = self._run(regular)
|
||||
|
||||
self.assertFalse(settled["changed"])
|
||||
@@ -454,7 +713,7 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
self.assertFalse(result["changed"])
|
||||
self.assertEqual(remote.calls, ["ansible.legacy.file_checksums"])
|
||||
|
||||
def test_missing_remote_directory_runs_the_regular_copy(self):
|
||||
def test_missing_remote_directory_is_ensured_in_one_batch(self):
|
||||
self._add_empty_directories()
|
||||
entries = self._with_leaves(0o755)
|
||||
del entries[self.DEST + "nested/.claude/.cc-writes"]
|
||||
@@ -463,18 +722,24 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
result = self._run(remote)
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertEqual(self._directory_calls(remote), self._leaf_paths())
|
||||
self.assertEqual(remote.calls, self.BATCH)
|
||||
self.assertEqual(
|
||||
self._batch_args(remote)["directories"], ["nested/.claude/.cc-writes"]
|
||||
)
|
||||
self.assertIsNone(self._batch_args(remote)["directory_mode"])
|
||||
|
||||
def test_directory_mode_drift_runs_the_regular_copy(self):
|
||||
def test_directory_mode_drift_is_settled_in_one_batch(self):
|
||||
self._add_empty_directories()
|
||||
remote = _FakeRemote(self._with_leaves(0o755))
|
||||
|
||||
result = self._run(remote, directory_mode="0750")
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertEqual(self._directory_calls(remote), self._leaf_paths())
|
||||
self.assertEqual(remote.calls, self.BATCH)
|
||||
self.assertEqual(self._batch_args(remote)["directories"], sorted(self.LEAVES))
|
||||
self.assertEqual(self._batch_args(remote)["directory_mode"], "0750")
|
||||
|
||||
def test_file_in_place_of_a_directory_runs_the_regular_copy(self):
|
||||
def test_file_in_place_of_a_directory_is_refused_then_fails_regularly(self):
|
||||
self._add_empty_directories()
|
||||
entries = self._with_leaves(0o755)
|
||||
entries[self.DEST + ".claude/.cc-writes"] = (b"", 0o644)
|
||||
@@ -482,6 +747,7 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
|
||||
result = self._run(remote)
|
||||
|
||||
self.assertEqual(remote.calls[:2], self.BATCH)
|
||||
self.assertTrue(result["failed"])
|
||||
self.assertIn(self.DEST + ".claude/.cc-writes", self._directory_calls(remote))
|
||||
|
||||
@@ -541,9 +807,10 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
result = self._run(remote)
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertIn("ansible.legacy.copy", remote.calls)
|
||||
self.assertEqual(remote.calls, self.BATCH)
|
||||
touched = [str(value) for args in remote.args for value in args.values()]
|
||||
self.assertEqual([value for value in touched if "__pycache__" in value], [])
|
||||
self.assertEqual(remote.transfers, [{"icon.png": b"png"}])
|
||||
|
||||
def test_bytecode_outside_a_cache_directory_stays_in_the_copy(self):
|
||||
(self.src / "legacy.pyc").write_bytes(b"bytecode")
|
||||
@@ -553,7 +820,7 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertIn(self.DEST + "legacy.pyc", remote.args[0]["paths"])
|
||||
self.assertIn("ansible.legacy.copy", remote.calls)
|
||||
self.assertEqual(remote.transfers, [{"legacy.pyc": b"bytecode"}])
|
||||
|
||||
def test_symlinked_root_passes_the_cache_filter_unchanged(self):
|
||||
with tempfile.TemporaryDirectory() as links:
|
||||
@@ -565,6 +832,79 @@ class TestStackHostCopyTree(unittest.TestCase):
|
||||
self.assertEqual(filtered["symlinks"], (str(self.src), "root"))
|
||||
|
||||
|
||||
class TestStackHostCopyFlatTree(_TreeHarness):
|
||||
FILES: ClassVar[dict[str, bytes]] = {
|
||||
f"slot-{index:02d}.png": f"palette-2-{index}".encode() for index in range(15)
|
||||
}
|
||||
|
||||
def _stale(self, names):
|
||||
entries = self._synced()
|
||||
entries.update({self.DEST + name: (b"palette-1", 0o644) for name in names})
|
||||
return entries
|
||||
|
||||
def _round_trips(self, entries, *, regular):
|
||||
remote = _FakeRemote(entries)
|
||||
if regular:
|
||||
with patch.object(ActionModule, "_write_tree", return_value=None):
|
||||
result = self._run(remote)
|
||||
else:
|
||||
result = self._run(remote)
|
||||
self.assertTrue(result["changed"])
|
||||
return {
|
||||
"modules": len(remote.calls),
|
||||
"transfers": len(remote.transfers),
|
||||
"fixups": remote.fixups,
|
||||
}
|
||||
|
||||
def test_all_files_changed_cost_one_transfer_and_two_module_calls(self):
|
||||
stale = self._stale(self.FILES)
|
||||
|
||||
before = self._round_trips(stale, regular=True)
|
||||
after = self._round_trips(stale, regular=False)
|
||||
|
||||
self.assertEqual(before, {"modules": 31, "transfers": 15, "fixups": 15})
|
||||
self.assertEqual(after, {"modules": 2, "transfers": 1, "fixups": 1})
|
||||
|
||||
def test_one_changed_file_costs_one_transfer_and_two_module_calls(self):
|
||||
stale = self._stale(["slot-07.png"])
|
||||
|
||||
before = self._round_trips(stale, regular=True)
|
||||
after = self._round_trips(stale, regular=False)
|
||||
|
||||
self.assertEqual(before, {"modules": 31, "transfers": 1, "fixups": 1})
|
||||
self.assertEqual(after, {"modules": 2, "transfers": 1, "fixups": 1})
|
||||
|
||||
def test_only_the_changed_file_is_sent(self):
|
||||
remote = _FakeRemote(self._stale(["slot-07.png"]))
|
||||
|
||||
self._run(remote)
|
||||
|
||||
self.assertEqual(remote.transfers, [{"slot-07.png": b"palette-2-7"}])
|
||||
self.assertEqual(remote.entries, self._synced())
|
||||
|
||||
def test_follow_without_symlinks_takes_the_batch(self):
|
||||
remote = _FakeRemote(self._stale(["slot-00.png", "slot-01.png"]))
|
||||
|
||||
result = self._run(remote, follow=True)
|
||||
|
||||
self.assertTrue(result["changed"])
|
||||
self.assertEqual(remote.calls, self.BATCH)
|
||||
|
||||
def test_follow_onto_a_symlinked_file_keeps_the_copy_semantics(self):
|
||||
entries = self._synced()
|
||||
entries[self.DEST + "slot-00.png"] = _Link(b"elsewhere")
|
||||
remote = _FakeRemote(entries)
|
||||
|
||||
self._run(remote, follow=True)
|
||||
|
||||
self.assertEqual(remote.calls[:2], self.BATCH)
|
||||
copies = self._copies(remote)
|
||||
self.assertEqual(
|
||||
[args["_original_basename"] for args in copies], ["slot-00.png"]
|
||||
)
|
||||
self.assertTrue(copies[0]["follow"])
|
||||
|
||||
|
||||
class TestStackHostCopyWalkSwap(unittest.TestCase):
|
||||
def _action(self):
|
||||
task = _FakeTask(args={"src": "x", "dest": "/opt/compose/app/x"})
|
||||
|
||||
Reference in new issue
Block a user