Files
computer-playbook/tests/integration/infrastructure/compose/test_no_raw_docker.py
T
kevinveenbirkenbachandClaude Opus 4.7 a5ba14d5b3 test(lint/cache-read): extend scope to all tests/ + convert callers
Widens the cache-read lint guard from ``tests/lint/`` to every ``.py``
under ``tests/``. Integration tests show the same compounding cost
pattern (one role's compose template / nginx conf / scrape fragment
read across many shape tests) that motivated the original lint for
lint/.

Converts 91 violations across 28 test files:

- **Cache (utils.cache.files.read_text)** for project-tracked reads
  and single-read tempdir fixtures, e.g. test_prometheus.py (32x),
  test_keycloak_import_flow.py (11x), test_dashboard_oidc.py (3x),
  test_changelog_md.py (8x), test_package_mirror.py (8x), plus
  test_logging.py / test_summarize.py / test_version_updater.py /
  test_get_url_retry.py / test_pull_specific_host.py.

- **Cache + UnicodeDecodeError handler** for project-walk scanners
  that previously used ``errors="replace"`` / ``errors="ignore"`` to
  defend against non-UTF-8 files (compose volumes, password quoting,
  template image, mailu dependency, URL reachability). Wraps each
  read in ``try/except (OSError, UnicodeDecodeError): continue``,
  which has the same observable effect (skip undecodable files).

- **Refactor away the binary heuristic** in test_no_raw_docker.py:
  ``read_bytes() + b"\x00" not in raw`` becomes
  ``read_text() / except UnicodeDecodeError``. Same files skipped,
  with cache benefit.

- **Variable extraction** in test_changelog_md.py / test_package_mirror.py
  for cases where ``self.assertEqual(path.read_text(...), expected)``
  exceeded ruff's 88-char limit. Ruff was wrapping the marker onto a
  line outside the AST span of the read_text call.

- **Compute-from-setup** for two test methods that previously read
  ``self.changelog`` twice (before/after) to verify a no-op:
  ``before`` is now derived from the known input, dropping the first
  read.

Five ``# nocheck: cache-read`` markers remain, each annotated inline
with the reason: idempotency tests where the second read must hit
disk to catch non-idempotent regressions, and tests with intermediate
``write_text`` whose subsequent read would return stale cache data.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-15 14:23:32 +02:00

284 lines
7.5 KiB
Python

from __future__ import annotations
import re
import subprocess
import unittest
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING
from utils.cache.files import iter_project_files, read_text
from . import PROJECT_ROOT
if TYPE_CHECKING:
from collections.abc import Sequence
@dataclass(frozen=True)
class Finding:
file: str
line_no: int
line: str
rule: str
suggestion: str
# Whitelist: ignore these file endings / filenames / path fragments
WHITELIST_SUFFIXES: tuple[str, ...] = (
".md",
".js",
".json",
".png",
".jpg",
".jpeg",
".gif",
".svg",
".ico",
".pdf",
".lock",
)
WHITELIST_FILENAMES: tuple[str, ...] = ("LICENSE",)
WHITELIST_PATH_FRAGMENTS: tuple[str, ...] = (
"/.git/",
"/.venv/",
"/.mypy_cache/",
"/.pytest_cache/",
"/node_modules/",
"/dist/",
"/build/",
"/.github/",
"/scripts/",
"/.tox/",
"/cli/",
"infinito_nexus.egg-info/",
)
# Optional: allow docker mentions in specific files (keep empty for strict)
WHITELIST_EXACT_PATHS: tuple[str, ...] = (
# "docs/somefile.txt",
)
# Only treat "docker ..." as a command when it appears in a command-like context.
# i.e. start of line or after common shell operators / subshell / command substitution.
_CMD_PREFIX = r"""
(?:
^\s* # line start
| [;&(]\s* # ; & (
| \|\s* # pipe
| &&\s* # &&
| \|\|\s* # ||
| \$(?:\(|\{)\s* # $( or ${
)
"""
# Optional sudo and optional absolute path to docker binary.
_DOCKER_BIN = r"(?:sudo\s+)?(?:/usr/bin/|/bin/|/usr/local/bin/)?docker"
_DOCKER_COMPOSE_BIN = r"(?:sudo\s+)?(?:/usr/bin/|/bin/|/usr/local/bin/)?docker-compose"
# Allowlist of real docker top-level subcommands that you consider "valid invocations".
# Extend when needed (keep it explicit to avoid false positives).
_DOCKER_SUBCOMMANDS = (
"run",
"exec",
"ps",
"inspect",
"logs",
"pull",
"push",
"build",
"login",
"logout",
"tag",
"rm",
"rmi",
"start",
"stop",
"restart",
"kill",
"cp",
"info",
"version",
"events",
"stats",
"system",
"container",
"image",
"volume",
"network",
"manifest",
"buildx",
"builder",
"context",
)
# Allowlist of compose verbs (docker compose <verb> / docker-compose <verb>)
_COMPOSE_VERBS = (
"up",
"down",
"pull",
"push",
"build",
"config",
"ps",
"logs",
"exec",
"run",
"start",
"stop",
"restart",
"rm",
"create",
"images",
"top",
)
# Compile patterns with VERBOSE for readability.
RE_DOCKER_CMD = re.compile(
rf"{_CMD_PREFIX}{_DOCKER_BIN}\s+(?:{'|'.join(map(re.escape, _DOCKER_SUBCOMMANDS))})\b",
re.IGNORECASE | re.VERBOSE,
)
RE_DOCKER_COMPOSE_CMD = re.compile(
rf"{_CMD_PREFIX}{_DOCKER_BIN}\s+compose\s+(?:{'|'.join(map(re.escape, _COMPOSE_VERBS))})\b",
re.IGNORECASE | re.VERBOSE,
)
RE_DOCKER_DASH_COMPOSE_CMD = re.compile(
rf"{_CMD_PREFIX}{_DOCKER_COMPOSE_BIN}\s+(?:{'|'.join(map(re.escape, _COMPOSE_VERBS))})\b",
re.IGNORECASE | re.VERBOSE,
)
# Rules: order matters (prefer specific messages)
RULES: tuple[tuple[str, re.Pattern, str], ...] = (
(
"docker compose usage",
RE_DOCKER_COMPOSE_CMD,
"Use 'compose <verb> ...' instead of 'docker compose <verb> ...'.",
),
(
"docker-compose usage",
RE_DOCKER_DASH_COMPOSE_CMD,
"Use 'compose <verb> ...' instead of 'docker-compose <verb> ...'.",
),
(
"docker CLI usage",
RE_DOCKER_CMD,
"Use 'container <cmd> ...' instead of calling 'docker <cmd> ...' directly.",
),
)
def git_ls_files(root: Path) -> list[Path]:
try:
out = subprocess.check_output(
["git", "-C", str(root), "ls-files", "-z"],
stderr=subprocess.STDOUT,
)
rels = [p for p in out.decode("utf-8", errors="replace").split("\0") if p]
return [root / p for p in rels]
except Exception:
# Fallback to the cached project walk so the lint guard does not
# need to step around a raw os.walk here. The whitelist filter is
# applied per-file (instead of pruning directories) which is
# functionally identical because the only things we pruned in the
# original walk were entire whitelisted subtrees.
results: list[Path] = []
for path_str in iter_project_files():
try:
rel = Path(path_str).relative_to(root).as_posix()
except ValueError:
continue
if any(fragment in f"/{rel}/" for fragment in WHITELIST_PATH_FRAGMENTS):
continue
results.append(Path(path_str))
return results
def is_whitelisted(path: Path, root: Path) -> bool:
rel = path.relative_to(root).as_posix()
if rel in WHITELIST_EXACT_PATHS:
return True
if path.name in WHITELIST_FILENAMES:
return True
if any(rel.endswith(suf) for suf in WHITELIST_SUFFIXES):
return True
rel_wrapped = f"/{rel}/"
return bool(any(fragment in rel_wrapped for fragment in WHITELIST_PATH_FRAGMENTS))
def scan_file(path: Path, root: Path) -> list[Finding]:
findings: list[Finding] = []
rel = path.relative_to(root).as_posix()
try:
text = read_text(str(path))
except (OSError, UnicodeDecodeError):
return findings
for idx, line in enumerate(text.splitlines(), start=1):
# Only flag when it looks like a command invocation.
for rule_name, pattern, suggestion in RULES:
if pattern.search(line):
findings.append(
Finding(
file=rel,
line_no=idx,
line=line.rstrip("\n"),
rule=rule_name,
suggestion=suggestion,
)
)
break
return findings
def format_findings(findings: Sequence[Finding]) -> str:
lines: list[str] = []
lines.append("Forbidden raw Docker command invocations detected.")
lines.append("")
lines.append("Why this matters:")
lines.append(
"- We enforce a convenience wrapper ('container' / 'compose') so the container engine can be switched quickly"
)
lines.append(
" (e.g., Docker -> Podman) without refactoring command strings across the repo."
)
lines.append("")
lines.append("Fix rules:")
lines.append("- 'docker <cmd> ...' -> 'container <cmd> ...'")
lines.append("- 'docker compose <verb> ...' -> 'compose <verb> ...'")
lines.append("- 'docker-compose <verb> ...' -> 'compose <verb> ...'")
lines.append("")
lines.append("Findings:")
for f in findings:
lines.append(f"- {f.file}:{f.line_no}: {f.line.strip()}")
lines.append(f" -> {f.suggestion}")
return "\n".join(lines)
class TestNoRawDockerCommands(unittest.TestCase):
def test_no_raw_docker_commands_in_repo(self) -> None:
root = PROJECT_ROOT
files = git_ls_files(root)
findings: list[Finding] = []
for p in files:
if not p.is_file():
continue
if is_whitelisted(p, root):
continue
findings.extend(scan_file(p, root))
if findings:
self.fail(format_findings(findings))
if __name__ == "__main__":
unittest.main()