mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-11 13:17:07 +00:00
kevinveenbirkenbach
76b139c894
Cleans up the lint debt the main->feature/web-app-kix merge surfaced.
Each change addresses a real failure surfaced by `make test`; no
``# nocheck`` blanket suppressions added.
Migrations to the shared cache helpers:
- ~25 test files swap raw ``yaml.safe_load`` / ``yaml.safe_dump`` /
``yaml.dump`` for ``utils.cache.yaml.{load_yaml_any,load_yaml_all,
load_yaml_str,dump_yaml,load_yaml_all_str}``.
- ~30 test files swap raw ``glob.glob`` / ``os.walk`` / ``Path.rglob``
for ``utils.cache.files.iter_project_files`` (or
``iter_non_ignored_files`` for the dependabot fallback walker, or
per-role ``Path.iterdir`` / ``Path.glob`` where the scope is a
single subtree).
- 3 ``Path.read_text`` lint helpers now route through
``utils.cache.files.read_text``.
Variant + service-flag fixes:
- web-app-kix: shift its subnet from 192.168.105.80/28 to
192.168.105.128/28 to clear the collision with web-app-hugo and
svc-prx-openresty.
- web-app-fediwall: add the missing ``mastodon`` / ``friendica``
service flags referenced by ``meta/variants.yml``; pin the css /
logout / dashboard / matomo / prometheus literals with
``# nocheck: dynamic-flag`` and add a third "everything off"
variant so the variant-coverage matrix sees both true and false
pinning for every dynamic flag.
- web-app-magento, web-app-shopware, web-svc-xmpp: switch the
``run_after: web-app-keycloak`` matcher from a literal-false
oauth2/oidc opt-out to the ``"{{ 'web-app-keycloak' in group_names }}"``
dynamic form, and extend each role's ``meta/variants.yml`` with the
matching true/false (and shopware's LDAP-only) coverage.
- web-app-fusiondirectory, web-app-matrix: declare the
``matomo`` flag so their ``run_after: web-app-matomo`` resolves
through an explicit services entry; add matomo true/false to both
variants files.
- web-app-fediwall, web-app-mini-qr: add the
``# nocheck: logout`` opt-out directly above ``logout:`` so the
logout-dashboard guard accepts the legitimately-disabled state.
- ~30 ``services.yml`` files: insert
``# nocheck: dynamic-flag`` directly above the literal
``css`` / ``redis`` / ``view`` / ``worker`` / ``email`` / ``html``
/ ``mariadb`` / ``opensearch`` / ``talk`` / ``logout`` blocks that
the dynamic-flag guard surfaced.
Code-quality follow-ups uncovered by ruff:
- utils/cache/yaml.py: collapse ``load_yaml_all_str`` to a one-line
return + on-call ``# nocheck: direct-yaml`` so the lint accepts
the cache module's own ``yaml.safe_load_all`` invocation.
- utils/templating.py: replace the ``Optional[Any]`` import that
slipped in from main with the modern ``Any | None``; rewrite the
cleanup ``try/except/pass`` blocks as
``contextlib.suppress(Exception)`` so the imported
``contextlib`` module is consumed consistently.
- 7 new-from-main lint test files (test_inline_script_block_size,
test_no_inline_multiline_php_in_sh, test_no_lookup_config_jinja_default,
test_no_inline_multiline_sql, test_web_role_no_web_dependency,
test_shared_primary_entity_run_once, test_fediwall_active): drop
their local ``Path(__file__).parents[N]`` PROJECT_ROOT
computations and import the package-level constant instead.
- roles/web-app-moodle/templates/playwright.env.j2: drop the
unreferenced ``KEYCLOAK_REALM_NAME`` env key that the
playwright-env-keys-used guard flagged.
112 lines
3.7 KiB
Python
112 lines
3.7 KiB
Python
import fnmatch
|
|
import os
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from utils.cache.files import iter_project_files
|
|
from utils.cache.yaml import load_yaml_any
|
|
|
|
from . import PROJECT_ROOT
|
|
|
|
SECURITY_WORKFLOW_PATH = PROJECT_ROOT / ".github" / "workflows" / "security-codeql.yml"
|
|
|
|
# Hidden directories that are the only ones worth scanning (the cached walker
|
|
# already prunes .git/.venv/__pycache__/node_modules etc.).
|
|
SCANNED_HIDDEN_DIRS = {
|
|
".github",
|
|
}
|
|
|
|
# Mapping: CodeQL language name -> filename patterns
|
|
CODEQL_FILENAME_INDICATORS: dict[str, list[str]] = {
|
|
"c-cpp": ["*.c", "*.cc", "*.cpp", "*.cxx", "*.h", "*.hh", "*.hpp", "*.hxx"],
|
|
"csharp": ["*.cs"],
|
|
"go": ["*.go"],
|
|
"java-kotlin": ["*.java", "*.kt", "*.kts"],
|
|
"javascript-typescript": ["*.js", "*.jsx", "*.ts", "*.tsx", "*.mjs", "*.cjs"],
|
|
"python": ["*.py"],
|
|
"ruby": ["*.rb"],
|
|
"rust": ["*.rs"],
|
|
"swift": ["*.swift"],
|
|
}
|
|
|
|
# Mapping: CodeQL language name -> repository-relative path patterns
|
|
CODEQL_PATH_INDICATORS: dict[str, list[str]] = {
|
|
"actions": [
|
|
".github/workflows/*.yml",
|
|
".github/workflows/*.yaml",
|
|
"action.yml",
|
|
"action.yaml",
|
|
],
|
|
}
|
|
|
|
|
|
def _load_active_languages() -> set[str]:
|
|
data = load_yaml_any(str(SECURITY_WORKFLOW_PATH))
|
|
|
|
include = data["jobs"]["analyze"]["strategy"]["matrix"]["include"]
|
|
return {
|
|
entry["language"]
|
|
for entry in include
|
|
if isinstance(entry, dict) and "language" in entry
|
|
}
|
|
|
|
|
|
def _matching_languages(rel_file: str, filename: str) -> set[str]:
|
|
matched = {
|
|
language
|
|
for language, patterns in CODEQL_FILENAME_INDICATORS.items()
|
|
if any(fnmatch.fnmatch(filename, pattern) for pattern in patterns)
|
|
}
|
|
matched.update(
|
|
language
|
|
for language, patterns in CODEQL_PATH_INDICATORS.items()
|
|
if any(fnmatch.fnmatch(rel_file, pattern) for pattern in patterns)
|
|
)
|
|
return matched
|
|
|
|
|
|
class TestSecurityWorkflowCoverage(unittest.TestCase):
|
|
def test_codeql_languages_match_repository_content(self):
|
|
"""The CodeQL workflow must only enable languages that are actually
|
|
present in the repository, and it must not miss any detected language."""
|
|
|
|
detected_languages: set[str] = set()
|
|
|
|
for path in iter_project_files():
|
|
rel_file = os.path.relpath(path, PROJECT_ROOT)
|
|
# Skip top-level hidden directories that are not SCANNED_HIDDEN_DIRS
|
|
# (the cached walker already prunes the vendor/cache junk).
|
|
top = rel_file.split(os.sep, 1)[0]
|
|
if top.startswith(".") and top not in SCANNED_HIDDEN_DIRS:
|
|
continue
|
|
filename = Path(rel_file).name
|
|
detected_languages.update(_matching_languages(rel_file, filename))
|
|
|
|
active_languages = _load_active_languages()
|
|
missing_languages = sorted(detected_languages - active_languages)
|
|
extra_languages = sorted(active_languages - detected_languages)
|
|
|
|
problems: list[str] = []
|
|
if missing_languages:
|
|
problems.append(
|
|
"Detected languages missing from .github/workflows/security-codeql.yml:"
|
|
)
|
|
problems.extend(f" - {language}" for language in missing_languages)
|
|
|
|
if extra_languages:
|
|
problems.append(
|
|
"Active CodeQL languages without matching repository files:"
|
|
)
|
|
problems.extend(f" - {language}" for language in extra_languages)
|
|
|
|
if problems:
|
|
self.fail(
|
|
"The CodeQL workflow matrix does not match the repository content.\n"
|
|
"Enable missing languages or comment out inactive ones.\n\n"
|
|
+ "\n".join(problems)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|