mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-13 22:26:49 +00:00
Widens the cache-read lint guard from ``tests/lint/`` to every ``.py`` under ``tests/``. Integration tests show the same compounding cost pattern (one role's compose template / nginx conf / scrape fragment read across many shape tests) that motivated the original lint for lint/. Converts 91 violations across 28 test files: - **Cache (utils.cache.files.read_text)** for project-tracked reads and single-read tempdir fixtures, e.g. test_prometheus.py (32x), test_keycloak_import_flow.py (11x), test_dashboard_oidc.py (3x), test_changelog_md.py (8x), test_package_mirror.py (8x), plus test_logging.py / test_summarize.py / test_version_updater.py / test_get_url_retry.py / test_pull_specific_host.py. - **Cache + UnicodeDecodeError handler** for project-walk scanners that previously used ``errors="replace"`` / ``errors="ignore"`` to defend against non-UTF-8 files (compose volumes, password quoting, template image, mailu dependency, URL reachability). Wraps each read in ``try/except (OSError, UnicodeDecodeError): continue``, which has the same observable effect (skip undecodable files). - **Refactor away the binary heuristic** in test_no_raw_docker.py: ``read_bytes() + b"\x00" not in raw`` becomes ``read_text() / except UnicodeDecodeError``. Same files skipped, with cache benefit. - **Variable extraction** in test_changelog_md.py / test_package_mirror.py for cases where ``self.assertEqual(path.read_text(...), expected)`` exceeded ruff's 88-char limit. Ruff was wrapping the marker onto a line outside the AST span of the read_text call. - **Compute-from-setup** for two test methods that previously read ``self.changelog`` twice (before/after) to verify a no-op: ``before`` is now derived from the known input, dropping the first read. Five ``# nocheck: cache-read`` markers remain, each annotated inline with the reason: idempotency tests where the second read must hit disk to catch non-idempotent regressions, and tests with intermediate ``write_text`` whose subsequent read would return stale cache data. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
284 lines
7.5 KiB
Python
284 lines
7.5 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
import subprocess
|
|
import unittest
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
|
|
from utils.cache.files import iter_project_files, read_text
|
|
|
|
from . import PROJECT_ROOT
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Sequence
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Finding:
|
|
file: str
|
|
line_no: int
|
|
line: str
|
|
rule: str
|
|
suggestion: str
|
|
|
|
|
|
# Whitelist: ignore these file endings / filenames / path fragments
|
|
WHITELIST_SUFFIXES: tuple[str, ...] = (
|
|
".md",
|
|
".js",
|
|
".json",
|
|
".png",
|
|
".jpg",
|
|
".jpeg",
|
|
".gif",
|
|
".svg",
|
|
".ico",
|
|
".pdf",
|
|
".lock",
|
|
)
|
|
|
|
WHITELIST_FILENAMES: tuple[str, ...] = ("LICENSE",)
|
|
|
|
WHITELIST_PATH_FRAGMENTS: tuple[str, ...] = (
|
|
"/.git/",
|
|
"/.venv/",
|
|
"/.mypy_cache/",
|
|
"/.pytest_cache/",
|
|
"/node_modules/",
|
|
"/dist/",
|
|
"/build/",
|
|
"/.github/",
|
|
"/scripts/",
|
|
"/.tox/",
|
|
"/cli/",
|
|
"infinito_nexus.egg-info/",
|
|
)
|
|
|
|
# Optional: allow docker mentions in specific files (keep empty for strict)
|
|
WHITELIST_EXACT_PATHS: tuple[str, ...] = (
|
|
# "docs/somefile.txt",
|
|
)
|
|
|
|
# Only treat "docker ..." as a command when it appears in a command-like context.
|
|
# i.e. start of line or after common shell operators / subshell / command substitution.
|
|
_CMD_PREFIX = r"""
|
|
(?:
|
|
^\s* # line start
|
|
| [;&(]\s* # ; & (
|
|
| \|\s* # pipe
|
|
| &&\s* # &&
|
|
| \|\|\s* # ||
|
|
| \$(?:\(|\{)\s* # $( or ${
|
|
)
|
|
"""
|
|
|
|
# Optional sudo and optional absolute path to docker binary.
|
|
_DOCKER_BIN = r"(?:sudo\s+)?(?:/usr/bin/|/bin/|/usr/local/bin/)?docker"
|
|
_DOCKER_COMPOSE_BIN = r"(?:sudo\s+)?(?:/usr/bin/|/bin/|/usr/local/bin/)?docker-compose"
|
|
|
|
# Allowlist of real docker top-level subcommands that you consider "valid invocations".
|
|
# Extend when needed (keep it explicit to avoid false positives).
|
|
_DOCKER_SUBCOMMANDS = (
|
|
"run",
|
|
"exec",
|
|
"ps",
|
|
"inspect",
|
|
"logs",
|
|
"pull",
|
|
"push",
|
|
"build",
|
|
"login",
|
|
"logout",
|
|
"tag",
|
|
"rm",
|
|
"rmi",
|
|
"start",
|
|
"stop",
|
|
"restart",
|
|
"kill",
|
|
"cp",
|
|
"info",
|
|
"version",
|
|
"events",
|
|
"stats",
|
|
"system",
|
|
"container",
|
|
"image",
|
|
"volume",
|
|
"network",
|
|
"manifest",
|
|
"buildx",
|
|
"builder",
|
|
"context",
|
|
)
|
|
|
|
# Allowlist of compose verbs (docker compose <verb> / docker-compose <verb>)
|
|
_COMPOSE_VERBS = (
|
|
"up",
|
|
"down",
|
|
"pull",
|
|
"push",
|
|
"build",
|
|
"config",
|
|
"ps",
|
|
"logs",
|
|
"exec",
|
|
"run",
|
|
"start",
|
|
"stop",
|
|
"restart",
|
|
"rm",
|
|
"create",
|
|
"images",
|
|
"top",
|
|
)
|
|
|
|
# Compile patterns with VERBOSE for readability.
|
|
RE_DOCKER_CMD = re.compile(
|
|
rf"{_CMD_PREFIX}{_DOCKER_BIN}\s+(?:{'|'.join(map(re.escape, _DOCKER_SUBCOMMANDS))})\b",
|
|
re.IGNORECASE | re.VERBOSE,
|
|
)
|
|
|
|
RE_DOCKER_COMPOSE_CMD = re.compile(
|
|
rf"{_CMD_PREFIX}{_DOCKER_BIN}\s+compose\s+(?:{'|'.join(map(re.escape, _COMPOSE_VERBS))})\b",
|
|
re.IGNORECASE | re.VERBOSE,
|
|
)
|
|
|
|
RE_DOCKER_DASH_COMPOSE_CMD = re.compile(
|
|
rf"{_CMD_PREFIX}{_DOCKER_COMPOSE_BIN}\s+(?:{'|'.join(map(re.escape, _COMPOSE_VERBS))})\b",
|
|
re.IGNORECASE | re.VERBOSE,
|
|
)
|
|
|
|
# Rules: order matters (prefer specific messages)
|
|
RULES: tuple[tuple[str, re.Pattern, str], ...] = (
|
|
(
|
|
"docker compose usage",
|
|
RE_DOCKER_COMPOSE_CMD,
|
|
"Use 'compose <verb> ...' instead of 'docker compose <verb> ...'.",
|
|
),
|
|
(
|
|
"docker-compose usage",
|
|
RE_DOCKER_DASH_COMPOSE_CMD,
|
|
"Use 'compose <verb> ...' instead of 'docker-compose <verb> ...'.",
|
|
),
|
|
(
|
|
"docker CLI usage",
|
|
RE_DOCKER_CMD,
|
|
"Use 'container <cmd> ...' instead of calling 'docker <cmd> ...' directly.",
|
|
),
|
|
)
|
|
|
|
|
|
def git_ls_files(root: Path) -> list[Path]:
|
|
try:
|
|
out = subprocess.check_output(
|
|
["git", "-C", str(root), "ls-files", "-z"],
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
rels = [p for p in out.decode("utf-8", errors="replace").split("\0") if p]
|
|
return [root / p for p in rels]
|
|
except Exception:
|
|
# Fallback to the cached project walk so the lint guard does not
|
|
# need to step around a raw os.walk here. The whitelist filter is
|
|
# applied per-file (instead of pruning directories) which is
|
|
# functionally identical because the only things we pruned in the
|
|
# original walk were entire whitelisted subtrees.
|
|
results: list[Path] = []
|
|
for path_str in iter_project_files():
|
|
try:
|
|
rel = Path(path_str).relative_to(root).as_posix()
|
|
except ValueError:
|
|
continue
|
|
if any(fragment in f"/{rel}/" for fragment in WHITELIST_PATH_FRAGMENTS):
|
|
continue
|
|
results.append(Path(path_str))
|
|
return results
|
|
|
|
|
|
def is_whitelisted(path: Path, root: Path) -> bool:
|
|
rel = path.relative_to(root).as_posix()
|
|
|
|
if rel in WHITELIST_EXACT_PATHS:
|
|
return True
|
|
if path.name in WHITELIST_FILENAMES:
|
|
return True
|
|
if any(rel.endswith(suf) for suf in WHITELIST_SUFFIXES):
|
|
return True
|
|
|
|
rel_wrapped = f"/{rel}/"
|
|
return bool(any(fragment in rel_wrapped for fragment in WHITELIST_PATH_FRAGMENTS))
|
|
|
|
|
|
def scan_file(path: Path, root: Path) -> list[Finding]:
|
|
findings: list[Finding] = []
|
|
rel = path.relative_to(root).as_posix()
|
|
|
|
try:
|
|
text = read_text(str(path))
|
|
except (OSError, UnicodeDecodeError):
|
|
return findings
|
|
|
|
for idx, line in enumerate(text.splitlines(), start=1):
|
|
# Only flag when it looks like a command invocation.
|
|
for rule_name, pattern, suggestion in RULES:
|
|
if pattern.search(line):
|
|
findings.append(
|
|
Finding(
|
|
file=rel,
|
|
line_no=idx,
|
|
line=line.rstrip("\n"),
|
|
rule=rule_name,
|
|
suggestion=suggestion,
|
|
)
|
|
)
|
|
break
|
|
|
|
return findings
|
|
|
|
|
|
def format_findings(findings: Sequence[Finding]) -> str:
|
|
lines: list[str] = []
|
|
lines.append("Forbidden raw Docker command invocations detected.")
|
|
lines.append("")
|
|
lines.append("Why this matters:")
|
|
lines.append(
|
|
"- We enforce a convenience wrapper ('container' / 'compose') so the container engine can be switched quickly"
|
|
)
|
|
lines.append(
|
|
" (e.g., Docker -> Podman) without refactoring command strings across the repo."
|
|
)
|
|
lines.append("")
|
|
lines.append("Fix rules:")
|
|
lines.append("- 'docker <cmd> ...' -> 'container <cmd> ...'")
|
|
lines.append("- 'docker compose <verb> ...' -> 'compose <verb> ...'")
|
|
lines.append("- 'docker-compose <verb> ...' -> 'compose <verb> ...'")
|
|
lines.append("")
|
|
lines.append("Findings:")
|
|
for f in findings:
|
|
lines.append(f"- {f.file}:{f.line_no}: {f.line.strip()}")
|
|
lines.append(f" -> {f.suggestion}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
class TestNoRawDockerCommands(unittest.TestCase):
|
|
def test_no_raw_docker_commands_in_repo(self) -> None:
|
|
root = PROJECT_ROOT
|
|
files = git_ls_files(root)
|
|
|
|
findings: list[Finding] = []
|
|
for p in files:
|
|
if not p.is_file():
|
|
continue
|
|
if is_whitelisted(p, root):
|
|
continue
|
|
findings.extend(scan_file(p, root))
|
|
|
|
if findings:
|
|
self.fail(format_findings(findings))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|