mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 02:54:26 +00:00
refactor(update): unify version bumpers under utils/update + cli/contributing/update + scripts/github/update
Folds the parallel Docker-image and git-ref version-update plumbing
into a single subsystem so the auto-update workflow, the external
freshness tests, and the unit tests all share one module layout.
utils/
update/
base.py # semver primitives shared by both backends
docker.py # OCI registry lookups + version: line rewriter
repository.py # git ls-remote walker + ref: line rewriter
cli/contributing/update/
docker/ # python -m cli.contributing.update.docker
repository/ # python -m cli.contributing.update.repository
scripts/github/update/
image_versions.sh
repository_refs.sh
Tests mirror the same shape under tests/{unit,external}/update/{docker,repository}/.
Also widens the Docker bump scope to all roles (previously gated on
`web-*`) so non-web roles like sys-svc-container get bumped too;
pins Bookwyrm's python image at 3.11-bookworm via
`# nocheck: docker-version` with a TODO until upstream supports
Python >= 3.12.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
kevinveenbirkenbachandClaude Opus 4.7 committed
1 parent
7c1ba22218
commit
98457f98fd
32 files changed
+898
-381
No files matched your search
@@ -30,7 +30,7 @@ jobs:
|
||||
run: bash scripts/github/install_update_python.sh
|
||||
|
||||
- name: Update Docker image versions
|
||||
run: bash scripts/github/update_docker_image_versions.sh
|
||||
run: bash scripts/github/update/image_versions.sh
|
||||
|
||||
- name: Check for changes
|
||||
id: diff
|
||||
@@ -43,7 +43,41 @@ jobs:
|
||||
UPDATE_BRANCH_PREFIX: update/docker-image-versions
|
||||
UPDATE_COMMIT_MESSAGE: "update(images): update Docker image versions"
|
||||
UPDATE_PR_TITLE: "update(images): update Docker image versions"
|
||||
UPDATE_PR_BODY: "Automated update of semver-based Docker image versions in `roles/web-*/config/main.yml`."
|
||||
UPDATE_PR_BODY: "Automated update of semver-based Docker image versions in `roles/*/meta/services.yml`."
|
||||
UPDATE_BASE_BRANCH: main
|
||||
run: bash scripts/github/open_update_pr.sh roles
|
||||
|
||||
update-repository-refs:
|
||||
name: 🌿 Update repository refs
|
||||
if: vars.CI_ENABLE_AUTO_UPDATES == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install update dependencies
|
||||
run: bash scripts/github/install_update_python.sh
|
||||
|
||||
- name: Update repository refs
|
||||
run: bash scripts/github/update/repository_refs.sh
|
||||
|
||||
- name: Check for changes
|
||||
id: diff
|
||||
run: bash scripts/github/check_git_changes.sh roles
|
||||
|
||||
- name: Open PR if repository refs changed
|
||||
if: steps.diff.outputs.changed == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
UPDATE_BRANCH_PREFIX: update/repository-refs
|
||||
UPDATE_COMMIT_MESSAGE: "update(repos): bump git refs to latest semver tags"
|
||||
UPDATE_PR_TITLE: "update(repos): bump git refs to latest semver tags"
|
||||
UPDATE_PR_BODY: "Automated update of semver-based git `ref:` values in `roles/*/meta/services.yml`. Upstream tags resolved via `git ls-remote --tags`."
|
||||
UPDATE_BASE_BRANCH: main
|
||||
run: bash scripts/github/open_update_pr.sh roles
|
||||
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
# Docker 🐳
|
||||
|
||||
Repository-side Docker maintenance: scan role config files for declared image versions and bump them in lockstep with upstream releases.
|
||||
@@ -0,0 +1,3 @@
|
||||
# Update 🔄
|
||||
|
||||
Repository-side version maintenance: scan role configuration files for declared upstream versions and bump them in lockstep with new releases.
|
||||
File renamed without changes.
+2
-2
@@ -1,5 +1,5 @@
|
||||
"""CLI entry point package for Docker image version updates."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT: Path = Path(__file__).resolve().parents[4]
|
||||
|
||||
"""CLI entry point package for Docker image version updates."""
|
||||
+2
-2
@@ -4,14 +4,14 @@ from __future__ import annotations
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from utils.docker.version_updater import apply_updates, find_outdated_updates
|
||||
from utils.update.docker import apply_updates, find_outdated_updates
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Update semver-based Docker image versions in web-*/meta/services.yml."
|
||||
description="Update semver-based Docker image versions in roles/*/meta/services.yml."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repo-root",
|
||||
@@ -0,0 +1,5 @@
|
||||
"""CLI entry point package for git repository ref updates."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT: Path = Path(__file__).resolve().parents[4]
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from utils.update.repository import apply_updates, find_outdated_updates
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Update semver-based git `ref:` values in roles/*/meta/services.yml."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repo-root",
|
||||
type=Path,
|
||||
default=PROJECT_ROOT,
|
||||
help="Repository root to scan and update",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Print planned updates without modifying files",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
repo_root = args.repo_root.resolve()
|
||||
updates = find_outdated_updates(repo_root)
|
||||
|
||||
if not updates:
|
||||
print("Repository refs are already up to date.")
|
||||
return 0
|
||||
|
||||
print("Planned repository ref updates:")
|
||||
for update in updates:
|
||||
rel_path = update.entry.config_path.relative_to(repo_root)
|
||||
entity = ".".join(update.entry.entity_path) or "<root>"
|
||||
print(
|
||||
f"- {rel_path}:{update.entry.line} {update.entry.role}/{entity} "
|
||||
f"{update.entry.repository} {update.entry.ref} -> {update.latest}"
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
return 0
|
||||
|
||||
changed_paths = apply_updates(updates)
|
||||
if changed_paths:
|
||||
print("\nUpdated files:")
|
||||
for path in changed_paths:
|
||||
print(f"- {path.relative_to(repo_root)}")
|
||||
else:
|
||||
print("\nNo files changed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -15,7 +15,7 @@ GHCR, or other external services. Run them with `make test-external`.
|
||||
## Location 📁
|
||||
|
||||
- External tests MUST live under `tests/external/`.
|
||||
- You SHOULD group them by dependency family such as `tests/external/docker/`
|
||||
- You SHOULD group them by dependency family such as `tests/external/update/`
|
||||
or `tests/external/github/` when more than one test exists.
|
||||
- Every external test directory MUST contain `__init__.py` so repository-wide
|
||||
test structure checks keep passing.
|
||||
|
||||
@@ -44,7 +44,7 @@ The placement rule is per check. The catalog "Position" column uses these labels
|
||||
| Rule | Position | Affected test | Effect |
|
||||
| --------------------------- | ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `url` | same or above | [test_urls_reachable.py](../../../../tests/external/repository/test_urls_reachable.py) | Skips probing the literal HTTP(S) URL. Use for CDN roots and API base URLs that 4xx without a path. |
|
||||
| `docker-version` | line above | [test_image_versions.py](../../../../tests/external/docker/test_image_versions.py) | Skips the live version-update warning for that image's `version:` key. |
|
||||
| `docker-version` | line above | [test_image_versions.py](../../../../tests/external/update/docker/test_image_versions.py) | Skips the live version-update warning for that image's `version:` key. |
|
||||
| `direct-yaml` | same line / span | [test_no_direct_calls.py](../../../../tests/lint/repository/yaml/test_no_direct_calls.py) | Allows the marked `yaml.safe_load` / `safe_dump` call to bypass `utils.cache.yaml`. MUST use `nocheck:`, not `noqa:`. |
|
||||
| `redundant-bool-ternary` | same line | [test_no_redundant_bool_patterns.py](../../../../tests/lint/repository/python/test_no_redundant_bool_patterns.py) | Allows the redundant `\| bool \| ternary('true','false')` pattern when a consumer requires the lowercase tokens. |
|
||||
| `redundant-bool-comparison` | same line | [test_no_redundant_bool_patterns.py](../../../../tests/lint/repository/python/test_no_redundant_bool_patterns.py) | Allows `... \| bool == True/False/true/false` when a real tri-state guard is intended (distinguishing `None` from `False`). |
|
||||
|
||||
@@ -91,7 +91,7 @@ Those images MUST be declared under `defaults/main.yml` and MUST be consumed via
|
||||
|
||||
[image_discovery.py](../../../../../utils/docker/image/discovery.py) enumerates all role images from both sources above.
|
||||
It is used by the mirror pipeline (`cli/contributing/mirror/`) and the external version-check test
|
||||
([`tests/external/docker/test_image_versions.py`](../../../../../tests/external/docker/test_image_versions.py)).
|
||||
([`tests/external/update/docker/test_image_versions.py`](../../../../../tests/external/update/docker/test_image_versions.py)).
|
||||
|
||||
## When `Dockerfile.j2` Is Acceptable 🤔
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ Use this rule when a role needs a Docker image definition.
|
||||
|
||||
### Image Versions 🏷️
|
||||
|
||||
Use this rule when a role pins container image versions in `roles/web-*/meta/services.yml`.
|
||||
Use this rule when a role pins container image versions in `roles/*/meta/services.yml`.
|
||||
|
||||
- You MUST keep semver-like image versions current when the upstream image publishes matching newer tags.
|
||||
- You MUST use `# nocheck: docker-version` directly above a `version:` key only for intentional exceptions that should not be flagged by the external image-version check.
|
||||
|
||||
@@ -291,7 +291,7 @@ the old `compose.…` or top-level paths.
|
||||
deep-merge over the assembled payload, just as they used to over
|
||||
`config/main.yml`.)
|
||||
- `utils/cache/base.py`
|
||||
- `utils/docker/version_updater.py`
|
||||
- `utils/update/docker.py`
|
||||
- `utils/docker/image/discovery.py`
|
||||
- `plugins/filter/native_metrics_target.py`
|
||||
- `plugins/lookup/config.py` (the `config` lookup itself)
|
||||
@@ -304,7 +304,7 @@ the old `compose.…` or top-level paths.
|
||||
- `plugins/lookup/nginx.py`
|
||||
- `plugins/lookup/application_gid.py`
|
||||
- `plugins/lookup/rbac_group_path.py`
|
||||
- `cli/contributing/docker/update_versions/__main__.py`
|
||||
- `cli/contributing/update/docker/__main__.py`
|
||||
- `cli/administration/deploy/development/inventory.py`
|
||||
- `cli/meta/roles/applications/resolution/combined/role_introspection.py`
|
||||
- `utils/roles/applications/services/resolver.py`
|
||||
|
||||
@@ -39,6 +39,8 @@ redis:
|
||||
bookwyrm:
|
||||
name: bookwyrm
|
||||
image: python
|
||||
# TODO: bump when upstream Bookwyrm declares support for Python >= 3.12 in requirements.txt
|
||||
# nocheck: docker-version
|
||||
version: 3.11-bookworm
|
||||
repository: https://github.com/bookwyrm-social/bookwyrm.git
|
||||
ref: v0.8.5
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# GitHub Workflow Helpers 🔧
|
||||
|
||||
This is the SPOT for GitHub workflow helper scripts under `scripts/github/`.
|
||||
These helpers are called from `.github/workflows/`.
|
||||
This directory holds the GitHub workflow helper scripts that the workflows under `.github/workflows/` call.
|
||||
For the repository Make target index, see [make.md](../../docs/contributing/tools/make.md).
|
||||
|
||||
## Entry Points 📋
|
||||
@@ -13,4 +12,5 @@ For the repository Make target index, see [make.md](../../docs/contributing/tool
|
||||
| `check_git_changes.sh` | Emits a `changed=true/false` workflow output based on `git diff`. | `.github/workflows/update.yml` |
|
||||
| `install_update_python.sh` | Installs Python dependencies needed by the automated update workflows. | `.github/workflows/update.yml` |
|
||||
| `open_update_pr.sh` | Commits update changes, pushes the branch, and creates or refreshes the matching Pull Request. | `.github/workflows/update.yml` |
|
||||
| `update_docker_image_versions.sh` | Runs the Docker image version updater CLI from GitHub Actions. | `.github/workflows/update.yml` |
|
||||
| `update/image_versions.sh` | Runs the Docker image version updater CLI from GitHub Actions. | `.github/workflows/update.yml` |
|
||||
| `update/repository_refs.sh` | Runs the git repository ref updater CLI from GitHub Actions. | `.github/workflows/update.yml` |
|
||||
@@ -0,0 +1,44 @@
|
||||
# Automated Update Helpers 🔄
|
||||
|
||||
This directory contains shell shims that the scheduled `Update: Versions` workflow at [update.yml](../../../.github/workflows/update.yml) invokes once per day to bump pinned upstream versions.
|
||||
|
||||
Each script is a thin wrapper around a Python CLI under [cli/contributing/](../../../cli/contributing/).
|
||||
The CLI does the actual scanning, version comparison, and in-place rewrite of `roles/*/meta/services.yml`; the workflow then checks for a diff and, if any file changed, opens or refreshes a pull request against `main` via [open_update_pr.sh](../open_update_pr.sh).
|
||||
|
||||
## Scripts 📄
|
||||
|
||||
### `image_versions.sh` 🐳
|
||||
|
||||
Bumps semver-based OCI image versions declared as `image:` + `version:` pairs under `roles/*/meta/services.yml`.
|
||||
|
||||
Delegates to [docker](../../../cli/contributing/update/docker/), which queries Docker Hub and GHCR for newer tags that share the current tag's depth and flavor.
|
||||
Tags that carry a `# nocheck: docker-version` marker above their `version:` key are skipped.
|
||||
|
||||
### `repository_refs.sh` 🌿
|
||||
|
||||
Bumps semver-based git refs declared as `repository:` + `ref:` pairs under `roles/*/meta/services.yml`, at any nesting depth (top-level entity, sub-entity, plugin map).
|
||||
|
||||
Delegates to [repository](../../../cli/contributing/update/repository/), which resolves upstream tags via `git ls-remote --tags <repository>`.
|
||||
Non-semver refs (branches like `master`, `main`, `stable`, commit SHAs) are skipped automatically.
|
||||
Refs that carry a `# nocheck: repository-version` marker above their `ref:` key are skipped.
|
||||
|
||||
## Conventions 📐
|
||||
|
||||
Each shim MUST:
|
||||
|
||||
- Resolve the repository root relative to its own location so it can run from any working directory.
|
||||
- Forward every argument it receives to the underlying CLI so `--dry-run` and `--repo-root` keep working from the shell.
|
||||
- Exit non-zero on CLI failure so the workflow surfaces the error.
|
||||
|
||||
Local smoke-test before changing a shim:
|
||||
|
||||
```bash
|
||||
bash scripts/github/update/image_versions.sh --dry-run
|
||||
bash scripts/github/update/repository_refs.sh --dry-run
|
||||
```
|
||||
|
||||
## Related 🔗
|
||||
|
||||
- [README.md](../README.md) for the broader inventory of GitHub workflow helpers.
|
||||
- [open_update_pr.sh](../open_update_pr.sh) for the shared PR-open/refresh logic both update jobs share.
|
||||
- [check_git_changes.sh](../check_git_changes.sh) for the diff probe that gates the PR step.
|
||||
+2
-2
@@ -2,7 +2,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)"
|
||||
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd)"
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
python3 -m cli.contributing.docker.update_versions "$@"
|
||||
python3 -m cli.contributing.update.docker "$@"
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd)"
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
python3 -m cli.contributing.update.repository "$@"
|
||||
@@ -1,251 +0,0 @@
|
||||
"""Check git `ref:` values in `roles/*/meta/services.yml`.
|
||||
|
||||
For every entity (at any depth — top-level entity, sub-entity, plugin
|
||||
map) that declares BOTH `repository:` and `ref:`, and whose `ref:` is
|
||||
a semver-compatible tag, this test fetches the upstream repository's
|
||||
git tags via ``git ls-remote --tags <repository>`` and warns when a
|
||||
newer semver tag exists.
|
||||
|
||||
Counterpart to ``tests/external/docker/test_image_versions.py`` —
|
||||
that one bumps OCI image tags, this one bumps git refs for
|
||||
from-source builds (plus plugin / build-helper / bootstrap repos).
|
||||
|
||||
External test: depends on live ``git ls-remote`` calls against the
|
||||
remotes declared in services.yml. The test always passes so normal
|
||||
validation stays stable even when remotes are slow or unreachable;
|
||||
outdated refs surface as warnings on stdout and as GitHub Actions
|
||||
``::warning::`` annotations on the offending services.yml line.
|
||||
|
||||
Suppress a check by placing ``# nocheck: repository-version`` on the
|
||||
line directly above the ``ref:`` key (blank lines between are
|
||||
ignored). Non-semver refs (`master`, `main`, `stable`, commit SHAs,
|
||||
…) are skipped automatically — only refs that match
|
||||
``utils.docker.version_updater.is_semver`` are checked.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import unittest
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from utils.annotations.message import warning
|
||||
from utils.annotations.suppress import is_suppressed_at
|
||||
from utils.cache.files import read_text
|
||||
from utils.cache.yaml import load_yaml_any
|
||||
from utils.docker.version_updater import (
|
||||
is_semver,
|
||||
latest_semver,
|
||||
version_depth,
|
||||
version_flavor,
|
||||
version_key,
|
||||
)
|
||||
from utils.roles.mapping import ROLE_FILE_META_SERVICES
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
_ROLES_ROOT = PROJECT_ROOT / "roles"
|
||||
_NOCHECK_MARKER = "repository-version"
|
||||
|
||||
|
||||
def _walk_repo_ref_pairs(node, path: list[str]) -> Iterator[tuple[list[str], str, str]]:
|
||||
"""Yield ``(entity_path, repository, ref)`` for every dict in the
|
||||
tree that declares both keys with truthy string values."""
|
||||
if isinstance(node, dict):
|
||||
repo = node.get("repository")
|
||||
ref = node.get("ref")
|
||||
if isinstance(repo, str) and repo and isinstance(ref, str) and ref:
|
||||
yield (list(path), repo.strip(), ref.strip())
|
||||
for key, value in node.items():
|
||||
if key in ("repository", "ref"):
|
||||
continue
|
||||
yield from _walk_repo_ref_pairs(value, [*path, str(key)])
|
||||
elif isinstance(node, list):
|
||||
for index, item in enumerate(node):
|
||||
yield from _walk_repo_ref_pairs(item, [*path, f"[{index}]"])
|
||||
|
||||
|
||||
def _suppressed_ref_paths(config_path: Path) -> set[tuple[int, int]]:
|
||||
"""Return ``(start_line, indent)`` pairs whose `ref:` line is
|
||||
annotated with ``# nocheck: repository-version`` on the immediately
|
||||
preceding line. Paired with ``_locate_ref_lines`` to match entities."""
|
||||
raw = read_text(str(config_path))
|
||||
lines = raw.splitlines()
|
||||
suppressed: set[tuple[int, int]] = set()
|
||||
for index, line in enumerate(lines):
|
||||
stripped = line.lstrip()
|
||||
if not stripped.startswith(("ref:", "ref :")):
|
||||
continue
|
||||
if is_suppressed_at(lines, index + 1, _NOCHECK_MARKER, mode="line-above"):
|
||||
indent = len(line) - len(stripped)
|
||||
suppressed.add((index + 1, indent))
|
||||
return suppressed
|
||||
|
||||
|
||||
def _locate_ref_line(config_path: Path, ref_value: str) -> int | None:
|
||||
"""Return the 1-indexed line number of the `ref: <value>` declaration
|
||||
in *config_path* whose value matches *ref_value*. Returns ``None``
|
||||
when no exact match is found (best-effort annotation anchor)."""
|
||||
raw = read_text(str(config_path))
|
||||
for index, line in enumerate(raw.splitlines(), start=1):
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith(("ref:", "ref :")):
|
||||
continue
|
||||
try:
|
||||
_, value = stripped.split(":", 1)
|
||||
except ValueError:
|
||||
continue
|
||||
value = value.strip().strip("'\"")
|
||||
if value == ref_value:
|
||||
return index
|
||||
return None
|
||||
|
||||
|
||||
def _git_ls_remote_tags(url: str) -> list[str]:
|
||||
"""Return semver-compatible tag names from ``git ls-remote --tags``.
|
||||
Stripped of the ``refs/tags/`` prefix and ``^{}`` peeled-tag markers."""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["git", "ls-remote", "--tags", url],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return []
|
||||
if proc.returncode != 0:
|
||||
return []
|
||||
tags: set[str] = set()
|
||||
for line in proc.stdout.splitlines():
|
||||
parts = line.split("\t")
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
name = parts[1]
|
||||
if not name.startswith("refs/tags/"):
|
||||
continue
|
||||
name = name[len("refs/tags/") :]
|
||||
name = name.removesuffix("^{}")
|
||||
if name:
|
||||
tags.add(name)
|
||||
return sorted(tags)
|
||||
|
||||
|
||||
class TestRepositoryVersions(unittest.TestCase):
|
||||
"""Warn about outdated semver `ref:` values in `meta/services.yml`."""
|
||||
|
||||
def test_repository_refs_are_current(self) -> None:
|
||||
entries: list[dict] = []
|
||||
for role_dir in sorted(p for p in _ROLES_ROOT.iterdir() if p.is_dir()):
|
||||
services_path = role_dir / ROLE_FILE_META_SERVICES
|
||||
if not services_path.is_file():
|
||||
continue
|
||||
data = load_yaml_any(str(services_path), default_if_missing=None)
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
suppressed = _suppressed_ref_paths(services_path)
|
||||
for entity_path, repo, ref in _walk_repo_ref_pairs(data, []):
|
||||
if not is_semver(ref):
|
||||
continue
|
||||
line_no = _locate_ref_line(services_path, ref)
|
||||
if line_no and any(line_no == sl for sl, _ in suppressed):
|
||||
continue
|
||||
entries.append(
|
||||
{
|
||||
"role": role_dir.name,
|
||||
"entity": ".".join(entity_path),
|
||||
"repository": repo,
|
||||
"ref": ref,
|
||||
"config_path": str(services_path.relative_to(PROJECT_ROOT)),
|
||||
"line": line_no,
|
||||
}
|
||||
)
|
||||
|
||||
# Deduplicate `git ls-remote` calls per repository.
|
||||
repo_tags: dict[str, list[str]] = {}
|
||||
for entry in entries:
|
||||
url = entry["repository"]
|
||||
if url in repo_tags:
|
||||
continue
|
||||
repo_tags[url] = _git_ls_remote_tags(url)
|
||||
|
||||
outdated: list[dict] = []
|
||||
unchecked: list[dict] = []
|
||||
for entry in entries:
|
||||
tags = repo_tags.get(entry["repository"], [])
|
||||
if not tags:
|
||||
unchecked.append(entry)
|
||||
continue
|
||||
latest = latest_semver(
|
||||
tags,
|
||||
version_depth(entry["ref"]),
|
||||
version_flavor(entry["ref"]),
|
||||
)
|
||||
if latest and version_key(entry["ref"]) < version_key(latest):
|
||||
outdated.append({**entry, "latest": latest})
|
||||
|
||||
if outdated:
|
||||
col_w = (28, 30, 50, 12)
|
||||
header = (
|
||||
f"{'Role':<{col_w[0]}} {'Entity':<{col_w[1]}} "
|
||||
f"{'Repository':<{col_w[2]}} {'Current':<{col_w[3]}} Latest"
|
||||
)
|
||||
rows = "\n".join(
|
||||
f"{o['role']:<{col_w[0]}} {o['entity']:<{col_w[1]}} "
|
||||
f"{o['repository']:<{col_w[2]}} {o['ref']:<{col_w[3]}} {o['latest']}"
|
||||
for o in outdated
|
||||
)
|
||||
print(
|
||||
f"\n⚠️ Outdated repository refs:\n{header}\n{'-' * 140}\n{rows}\n"
|
||||
f"\n💡 To suppress a warning add above the ref: key:\n"
|
||||
f" # nocheck: repository-version"
|
||||
)
|
||||
for o in outdated:
|
||||
msg = (
|
||||
f"{o['role']}/{o['entity']}: {o['repository']} is at "
|
||||
f"{o['ref']}, latest semver tag is {o['latest']}"
|
||||
)
|
||||
warning(
|
||||
msg,
|
||||
title="Outdated repository ref",
|
||||
file=o["config_path"],
|
||||
line=o["line"],
|
||||
)
|
||||
|
||||
if unchecked:
|
||||
col_w = (28, 30, 50)
|
||||
header = (
|
||||
f"{'Role':<{col_w[0]}} {'Entity':<{col_w[1]}} "
|
||||
f"{'Repository':<{col_w[2]}} Current"
|
||||
)
|
||||
rows = "\n".join(
|
||||
f"{o['role']:<{col_w[0]}} {o['entity']:<{col_w[1]}} "
|
||||
f"{o['repository']:<{col_w[2]}} {o['ref']}"
|
||||
for o in unchecked
|
||||
)
|
||||
print(
|
||||
f"\n🔍 Unchecked repository refs (remote unreachable or no tags):\n"
|
||||
f"{header}\n{'-' * 130}\n{rows}"
|
||||
)
|
||||
for o in unchecked:
|
||||
msg = (
|
||||
f"{o['role']}/{o['entity']}: {o['repository']} ref "
|
||||
f"{o['ref']} could not be checked (remote unreachable)"
|
||||
)
|
||||
warning(
|
||||
msg,
|
||||
title="🔍 Unchecked repository ref",
|
||||
file=o["config_path"],
|
||||
line=o["line"],
|
||||
)
|
||||
|
||||
# Always pass — outdated refs are warnings, not hard failures.
|
||||
self.assertIsNotNone(entries)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
File renamed without changes.
+3
@@ -0,0 +1,3 @@
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT: Path = Path(__file__).resolve().parents[4]
|
||||
Vendored
+12
-13
@@ -1,4 +1,4 @@
|
||||
"""Check Docker image versions in roles/web-*/meta/services.yml.
|
||||
"""Check Docker image versions in roles/*/meta/services.yml.
|
||||
|
||||
For each service with a semver-compatible version tag the latest available
|
||||
tag on Docker Hub is fetched and compared. Outdated versions are reported as
|
||||
@@ -31,19 +31,21 @@ import unittest
|
||||
|
||||
from utils.annotations.message import warning
|
||||
from utils.docker.image.discovery import iter_role_images
|
||||
from utils.docker.version_updater import (
|
||||
fetch_dockerhub_tags,
|
||||
fetch_ghcr_tags,
|
||||
is_dockerhub,
|
||||
is_ghcr,
|
||||
from utils.roles.mapping import ROLE_FILE_META_SERVICES
|
||||
from utils.update.base import (
|
||||
is_semver,
|
||||
latest_semver,
|
||||
suppressed_services,
|
||||
version_depth,
|
||||
version_flavor,
|
||||
version_key,
|
||||
)
|
||||
from utils.roles.mapping import ROLE_FILE_META_SERVICES
|
||||
from utils.update.docker import (
|
||||
fetch_dockerhub_tags,
|
||||
fetch_ghcr_tags,
|
||||
is_dockerhub,
|
||||
is_ghcr,
|
||||
suppressed_services,
|
||||
)
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
@@ -52,12 +54,9 @@ _ROLES_ROOT = _REPO_ROOT / "roles"
|
||||
|
||||
|
||||
def _collect_entries() -> list[dict]:
|
||||
"""Collect (role, service, image, version, config_path) for semver versions in web-* roles."""
|
||||
"""Collect (role, service, image, version, config_path) for semver versions across all roles."""
|
||||
entries: list[dict] = []
|
||||
for ref in iter_role_images(_REPO_ROOT):
|
||||
# Only web-* roles
|
||||
if not ref.role.startswith("web-"):
|
||||
continue
|
||||
# Only semver versions (pure or `<semver>-<flavor>`)
|
||||
if not is_semver(ref.version):
|
||||
continue
|
||||
@@ -108,7 +107,7 @@ def _emit_unchecked_annotation(
|
||||
|
||||
|
||||
class TestDockerImageVersions(unittest.TestCase):
|
||||
"""Warn about outdated live Docker image versions in roles/web-*/meta/services.yml."""
|
||||
"""Warn about outdated live Docker image versions in roles/*/meta/services.yml."""
|
||||
|
||||
def test_image_versions_are_current(self) -> None:
|
||||
entries = _collect_entries()
|
||||
@@ -0,0 +1,3 @@
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT: Path = Path(__file__).resolve().parents[4]
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Warn about outdated semver git `ref:` values in `roles/*/meta/services.yml`.
|
||||
|
||||
For every entity (at any depth — top-level entity, sub-entity, plugin
|
||||
map) that declares BOTH `repository:` and `ref:`, and whose `ref:` is
|
||||
a semver-compatible tag, this test fetches the upstream repository's
|
||||
git tags via ``git ls-remote --tags <repository>`` and warns when a
|
||||
newer semver tag exists.
|
||||
|
||||
Counterpart to ``tests/external/update/docker/test_image_versions.py``
|
||||
(that one bumps OCI image tags, this one bumps git refs for
|
||||
from-source builds, plugin and build-helper repos).
|
||||
The shared discovery / version logic lives in
|
||||
:mod:`utils.update.repository`; the CI auto-update job at
|
||||
``.github/workflows/update.yml`` (``update-repository-refs``) consumes
|
||||
the same module to open PRs against ``main``.
|
||||
|
||||
External test: depends on live ``git ls-remote`` calls against the
|
||||
remotes declared in services.yml. The test always passes so normal
|
||||
validation stays stable even when remotes are slow or unreachable;
|
||||
outdated refs surface as warnings on stdout and as GitHub Actions
|
||||
``::warning::`` annotations on the offending services.yml line.
|
||||
|
||||
Suppress a check by placing ``# nocheck: repository-version`` on the
|
||||
line directly above the ``ref:`` key (blank lines between are
|
||||
ignored). Non-semver refs (``master``, ``main``, ``stable``, commit
|
||||
SHAs) are skipped automatically; only refs that match
|
||||
:func:`utils.update.base.is_semver` are checked.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from utils.annotations.message import warning
|
||||
from utils.update.repository import (
|
||||
collect_entries,
|
||||
find_outdated_updates,
|
||||
)
|
||||
|
||||
from . import PROJECT_ROOT
|
||||
|
||||
|
||||
class TestRepositoryVersions(unittest.TestCase):
|
||||
"""Warn about outdated semver `ref:` values in `meta/services.yml`."""
|
||||
|
||||
def test_repository_refs_are_current(self) -> None:
|
||||
entries = collect_entries(PROJECT_ROOT)
|
||||
updates = find_outdated_updates(PROJECT_ROOT)
|
||||
|
||||
if updates:
|
||||
col_w = (28, 30, 50, 12)
|
||||
header = (
|
||||
f"{'Role':<{col_w[0]}} {'Entity':<{col_w[1]}} "
|
||||
f"{'Repository':<{col_w[2]}} {'Current':<{col_w[3]}} Latest"
|
||||
)
|
||||
rows = "\n".join(
|
||||
f"{u.entry.role:<{col_w[0]}} "
|
||||
f"{'.'.join(u.entry.entity_path) or '<root>':<{col_w[1]}} "
|
||||
f"{u.entry.repository:<{col_w[2]}} "
|
||||
f"{u.entry.ref:<{col_w[3]}} {u.latest}"
|
||||
for u in updates
|
||||
)
|
||||
print(
|
||||
f"\n⚠️ Outdated repository refs:\n{header}\n{'-' * 140}\n{rows}\n"
|
||||
f"\n💡 To suppress a warning add above the ref: key:\n"
|
||||
f" # nocheck: repository-version"
|
||||
)
|
||||
for u in updates:
|
||||
entity = ".".join(u.entry.entity_path) or "<root>"
|
||||
msg = (
|
||||
f"{u.entry.role}/{entity}: {u.entry.repository} is at "
|
||||
f"{u.entry.ref}, latest semver tag is {u.latest}"
|
||||
)
|
||||
warning(
|
||||
msg,
|
||||
title="Outdated repository ref",
|
||||
file=str(u.entry.config_path.relative_to(PROJECT_ROOT)),
|
||||
line=u.entry.line,
|
||||
)
|
||||
|
||||
# Always pass — outdated refs are warnings, not hard failures.
|
||||
self.assertIsNotNone(entries)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Whitespace-only changes.
+2
-28
@@ -1,21 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from utils.cache.files import read_text
|
||||
from utils.docker.version_updater import (
|
||||
from utils.update.base import (
|
||||
is_semver,
|
||||
latest_semver,
|
||||
update_config_versions,
|
||||
version_depth,
|
||||
version_flavor,
|
||||
version_key,
|
||||
)
|
||||
|
||||
|
||||
class TestDockerVersionUpdater(unittest.TestCase):
|
||||
class TestUpdateBase(unittest.TestCase):
|
||||
def test_latest_semver_respects_depth(self) -> None:
|
||||
tags = ["4", "4.5", "4.6", "4.5.1", "5"]
|
||||
|
||||
@@ -62,28 +58,6 @@ class TestDockerVersionUpdater(unittest.TestCase):
|
||||
# Flavor-less current version MUST stay in the unflavored lane.
|
||||
self.assertEqual(latest_semver(tags, 3, ""), "5.4.7")
|
||||
|
||||
def test_update_config_versions_updates_only_target_services(self) -> None:
|
||||
# Per the file root of meta/services.yml IS the services map
|
||||
# (no `compose.services` envelope), so service keys are at indent 0.
|
||||
original = """moodle:
|
||||
version: "4.5" # Keep comment
|
||||
image: bitnamilegacy/moodle
|
||||
nginx:
|
||||
version: alpine
|
||||
image: nginx
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
config_path = Path(tmpdir) / "main.yml"
|
||||
config_path.write_text(original, encoding="utf-8")
|
||||
|
||||
changed = update_config_versions(config_path, {"moodle": "5.0"})
|
||||
|
||||
self.assertTrue(changed)
|
||||
updated = read_text(str(config_path))
|
||||
self.assertIn('version: "5.0" # Keep comment', updated)
|
||||
self.assertIn("version: alpine", updated)
|
||||
self.assertNotIn('version: "4.5" # Keep comment', updated)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,36 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from utils.cache.files import read_text
|
||||
from utils.update.docker import update_config_versions
|
||||
|
||||
|
||||
class TestUpdateDocker(unittest.TestCase):
|
||||
def test_update_config_versions_updates_only_target_services(self) -> None:
|
||||
# The file root of meta/services.yml IS the services map (no
|
||||
# `compose.services` envelope), so service keys are at indent 0.
|
||||
original = """moodle:
|
||||
version: "4.5" # Keep comment
|
||||
image: bitnamilegacy/moodle
|
||||
nginx:
|
||||
version: alpine
|
||||
image: nginx
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
config_path = Path(tmpdir) / "main.yml"
|
||||
config_path.write_text(original, encoding="utf-8")
|
||||
|
||||
changed = update_config_versions(config_path, {"moodle": "5.0"})
|
||||
|
||||
self.assertTrue(changed)
|
||||
updated = read_text(str(config_path))
|
||||
self.assertIn('version: "5.0" # Keep comment', updated)
|
||||
self.assertIn("version: alpine", updated)
|
||||
self.assertNotIn('version: "4.5" # Keep comment', updated)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,223 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from utils.cache.files import read_text
|
||||
from utils.update.repository import (
|
||||
RepositoryRefEntry,
|
||||
RepositoryRefUpdate,
|
||||
apply_updates,
|
||||
suppressed_ref_lines,
|
||||
update_config_refs,
|
||||
walk_repo_ref_pairs,
|
||||
)
|
||||
|
||||
|
||||
class TestWalkRepoRefPairs(unittest.TestCase):
|
||||
def test_finds_top_level_entity(self) -> None:
|
||||
data = {
|
||||
"bookwyrm": {
|
||||
"repository": "https://github.com/bookwyrm-social/bookwyrm.git",
|
||||
"ref": "v0.8.5",
|
||||
},
|
||||
}
|
||||
|
||||
pairs = list(walk_repo_ref_pairs(data, ()))
|
||||
|
||||
self.assertEqual(
|
||||
pairs,
|
||||
[
|
||||
(
|
||||
("bookwyrm",),
|
||||
"https://github.com/bookwyrm-social/bookwyrm.git",
|
||||
"v0.8.5",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
def test_descends_into_nested_plugin_map(self) -> None:
|
||||
# Discourse-style plugin nesting: discourse.plugins.<name>.repository+ref
|
||||
data = {
|
||||
"discourse": {
|
||||
"plugins": {
|
||||
"discourse-ldap-auth": {
|
||||
"repository": "https://github.com/jonmbake/discourse-ldap-auth.git",
|
||||
"ref": "master",
|
||||
},
|
||||
"discourse-prometheus": {
|
||||
"repository": "https://github.com/discourse/discourse-prometheus.git",
|
||||
"ref": "v1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
pairs = list(walk_repo_ref_pairs(data, ()))
|
||||
entity_paths = {pair[0] for pair in pairs}
|
||||
|
||||
self.assertIn(("discourse", "plugins", "discourse-ldap-auth"), entity_paths)
|
||||
self.assertIn(("discourse", "plugins", "discourse-prometheus"), entity_paths)
|
||||
|
||||
def test_skips_when_only_repository_or_only_ref(self) -> None:
|
||||
data = {
|
||||
"only-repo": {"repository": "https://example.invalid/x.git"},
|
||||
"only-ref": {"ref": "v1.0.0"},
|
||||
"both": {
|
||||
"repository": "https://example.invalid/y.git",
|
||||
"ref": "v2.0.0",
|
||||
},
|
||||
}
|
||||
|
||||
pairs = list(walk_repo_ref_pairs(data, ()))
|
||||
|
||||
self.assertEqual(len(pairs), 1)
|
||||
self.assertEqual(pairs[0][0], ("both",))
|
||||
|
||||
def test_walks_lists(self) -> None:
|
||||
data = {
|
||||
"repos": [
|
||||
{
|
||||
"repository": "https://example.invalid/a.git",
|
||||
"ref": "v1",
|
||||
},
|
||||
{
|
||||
"repository": "https://example.invalid/b.git",
|
||||
"ref": "v2",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
pairs = list(walk_repo_ref_pairs(data, ()))
|
||||
|
||||
self.assertEqual(len(pairs), 2)
|
||||
self.assertEqual(pairs[0][0], ("repos", "[0]"))
|
||||
self.assertEqual(pairs[1][0], ("repos", "[1]"))
|
||||
|
||||
|
||||
class TestUpdateConfigRefs(unittest.TestCase):
|
||||
def test_rewrites_target_line_and_preserves_others(self) -> None:
|
||||
original = """plugin-a:
|
||||
repository: https://example.invalid/a.git
|
||||
ref: v1.0.0
|
||||
plugin-b:
|
||||
repository: https://example.invalid/b.git
|
||||
ref: v2.0.0
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
config_path = Path(tmpdir) / "services.yml"
|
||||
config_path.write_text(original, encoding="utf-8")
|
||||
|
||||
# plugin-a's `ref:` is on line 3.
|
||||
changed = update_config_refs(config_path, {3: "v1.1.0"})
|
||||
|
||||
self.assertTrue(changed)
|
||||
updated = read_text(str(config_path))
|
||||
self.assertIn(" ref: v1.1.0\n", updated)
|
||||
self.assertIn(" ref: v2.0.0\n", updated)
|
||||
|
||||
def test_preserves_quoting_and_trailing_comment(self) -> None:
|
||||
original = ' ref: "v0.7.5" # pin\n'
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
config_path = Path(tmpdir) / "services.yml"
|
||||
config_path.write_text(original, encoding="utf-8")
|
||||
|
||||
changed = update_config_refs(config_path, {1: "v0.8.5"})
|
||||
|
||||
self.assertTrue(changed)
|
||||
self.assertEqual(
|
||||
read_text(str(config_path)),
|
||||
' ref: "v0.8.5" # pin\n',
|
||||
)
|
||||
|
||||
def test_no_op_when_target_value_already_matches(self) -> None:
|
||||
original = " ref: v1.0.0\n"
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
config_path = Path(tmpdir) / "services.yml"
|
||||
config_path.write_text(original, encoding="utf-8")
|
||||
|
||||
changed = update_config_refs(config_path, {1: "v1.0.0"})
|
||||
|
||||
self.assertFalse(changed)
|
||||
self.assertEqual(read_text(str(config_path)), original)
|
||||
|
||||
def test_skips_out_of_range_line_numbers(self) -> None:
|
||||
original = " ref: v1.0.0\n"
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
config_path = Path(tmpdir) / "services.yml"
|
||||
config_path.write_text(original, encoding="utf-8")
|
||||
|
||||
changed = update_config_refs(config_path, {99: "v2.0.0"})
|
||||
|
||||
self.assertFalse(changed)
|
||||
self.assertEqual(read_text(str(config_path)), original)
|
||||
|
||||
|
||||
class TestSuppressedRefLines(unittest.TestCase):
|
||||
def test_recognises_line_above_marker(self) -> None:
|
||||
content = """plugin-a:
|
||||
repository: https://example.invalid/a.git
|
||||
# nocheck: repository-version
|
||||
ref: v1.0.0
|
||||
plugin-b:
|
||||
repository: https://example.invalid/b.git
|
||||
ref: v2.0.0
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
config_path = Path(tmpdir) / "services.yml"
|
||||
config_path.write_text(content, encoding="utf-8")
|
||||
|
||||
suppressed = suppressed_ref_lines(config_path)
|
||||
|
||||
# plugin-a's `ref:` sits on line 4 and is suppressed.
|
||||
self.assertEqual(suppressed, {4})
|
||||
|
||||
|
||||
class TestApplyUpdates(unittest.TestCase):
|
||||
def test_groups_updates_per_config_file(self) -> None:
|
||||
file_a_content = " ref: v1.0.0\n"
|
||||
file_b_content = " ref: v2.0.0\n"
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
config_a = Path(tmpdir) / "a" / "services.yml"
|
||||
config_b = Path(tmpdir) / "b" / "services.yml"
|
||||
config_a.parent.mkdir()
|
||||
config_b.parent.mkdir()
|
||||
config_a.write_text(file_a_content, encoding="utf-8")
|
||||
config_b.write_text(file_b_content, encoding="utf-8")
|
||||
|
||||
updates = [
|
||||
RepositoryRefUpdate(
|
||||
entry=RepositoryRefEntry(
|
||||
role="role-a",
|
||||
entity_path=("plugin-a",),
|
||||
repository="https://example.invalid/a.git",
|
||||
ref="v1.0.0",
|
||||
config_path=config_a,
|
||||
line=1,
|
||||
),
|
||||
latest="v1.1.0",
|
||||
),
|
||||
RepositoryRefUpdate(
|
||||
entry=RepositoryRefEntry(
|
||||
role="role-b",
|
||||
entity_path=("plugin-b",),
|
||||
repository="https://example.invalid/b.git",
|
||||
ref="v2.0.0",
|
||||
config_path=config_b,
|
||||
line=1,
|
||||
),
|
||||
latest="v2.1.0",
|
||||
),
|
||||
]
|
||||
|
||||
changed = apply_updates(updates)
|
||||
|
||||
self.assertEqual(set(changed), {config_a, config_b})
|
||||
self.assertEqual(read_text(str(config_a)), " ref: v1.1.0\n")
|
||||
self.assertEqual(read_text(str(config_b)), " ref: v2.1.0\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Whitespace-only changes.
@@ -0,0 +1,65 @@
|
||||
"""Semver primitives shared by every version-bump backend.
|
||||
|
||||
A "version" here is a tag of the shape ``v?<numeric-semver>(-<flavor>)?``,
|
||||
where the numeric part has 1 to 4 dot-separated components and the
|
||||
optional ``-<flavor>`` suffix is an opaque discriminator (e.g. the
|
||||
Docker Official Image tag ``5.4.5-php8.3-apache`` or the npm-style
|
||||
``1.2.3-alpha``).
|
||||
|
||||
Upgrade candidates MUST share the same depth (1, 2, 3 or 4 components)
|
||||
AND the same flavor as the current tag, so that ``5.4.5-php8.3-apache``
|
||||
never silently bumps to ``5.4.6-php8.4-apache`` (different runtime) or
|
||||
to ``5.4.6`` (different depth).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_SEMVER_CORE = r"v?\d+(?:\.\d+){0,3}"
|
||||
_VERSIONED_TAG_RE = re.compile(rf"^(?P<semver>{_SEMVER_CORE})(?P<flavor>-\S+)?$")
|
||||
|
||||
|
||||
def _parse_versioned_tag(tag: str) -> tuple[str, str] | None:
|
||||
match = _VERSIONED_TAG_RE.match(str(tag).strip())
|
||||
if match is None:
|
||||
return None
|
||||
return match.group("semver"), match.group("flavor") or ""
|
||||
|
||||
|
||||
def is_semver(value: str) -> bool:
|
||||
return _parse_versioned_tag(value) is not None
|
||||
|
||||
|
||||
def version_key(tag: str) -> tuple[int, ...]:
|
||||
parsed = _parse_versioned_tag(tag)
|
||||
if parsed is None:
|
||||
return (0,) * 4
|
||||
semver, _flavor = parsed
|
||||
parts = tuple(int(part) for part in semver.lstrip("v").split("."))
|
||||
return parts + (0,) * (4 - len(parts))
|
||||
|
||||
|
||||
def version_depth(tag: str) -> int:
|
||||
parsed = _parse_versioned_tag(tag)
|
||||
if parsed is None:
|
||||
return 0
|
||||
semver, _flavor = parsed
|
||||
return len(semver.lstrip("v").split("."))
|
||||
|
||||
|
||||
def version_flavor(tag: str) -> str:
|
||||
"""Return the ``-<flavor>`` suffix of a versioned tag, or "" when none."""
|
||||
parsed = _parse_versioned_tag(tag)
|
||||
return parsed[1] if parsed else ""
|
||||
|
||||
|
||||
def latest_semver(tags: list[str], depth: int, flavor: str = "") -> str | None:
|
||||
candidates = [
|
||||
tag
|
||||
for tag in tags
|
||||
if is_semver(tag)
|
||||
and version_depth(tag) == depth
|
||||
and version_flavor(tag) == flavor
|
||||
]
|
||||
return max(candidates, key=version_key, default=None)
|
||||
@@ -1,3 +1,15 @@
|
||||
"""Detect and rewrite outdated Docker image `version:` tags in
|
||||
`roles/*/meta/services.yml`.
|
||||
|
||||
Counterpart to :mod:`utils.update.repository` (which handles git refs).
|
||||
Both share the semver primitives in :mod:`utils.update.base`; this
|
||||
module owns the Docker-Hub / GHCR registry lookups and the YAML
|
||||
walker that rewrites the matching ``version:`` line.
|
||||
|
||||
Suppress a check by placing ``# nocheck: docker-version`` on the line
|
||||
directly above the ``version:`` key.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
@@ -20,19 +32,22 @@ from utils.docker.image.ref import (
|
||||
split_registry_and_name,
|
||||
)
|
||||
from utils.roles.mapping import ROLE_FILE_META_SERVICES
|
||||
from utils.update.base import (
|
||||
is_semver,
|
||||
latest_semver,
|
||||
version_depth,
|
||||
version_flavor,
|
||||
version_key,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
_SEMVER_CORE = r"v?\d+(?:\.\d+){0,3}"
|
||||
# Tags that extend a semver with a `-<flavor>` suffix, e.g. the Docker
|
||||
# Official image tag `5.4.5-php8.3-apache`. The flavor is treated as an
|
||||
# opaque discriminator: tags are only considered upgrade candidates for
|
||||
# each other when their flavor strings match.
|
||||
_VERSIONED_TAG_RE = re.compile(rf"^(?P<semver>{_SEMVER_CORE})(?P<flavor>-\S+)?$")
|
||||
_KEY_RE = re.compile(r"^(?P<indent>\s*)(?P<key>[A-Za-z0-9_-]+):(?P<rest>.*)$")
|
||||
_VERSION_VALUE_RE = re.compile(
|
||||
r"^(?P<prefix>\s*version\s*:\s*)(?P<quote>[\"']?)(?P<value>[^\"'#\s]+)(?P=quote)(?P<suffix>\s*(?:#.*)?)$"
|
||||
r"^(?P<prefix>\s*version\s*:\s*)"
|
||||
r"(?P<quote>[\"']?)(?P<value>[^\"'#\s]+)(?P=quote)"
|
||||
r"(?P<suffix>\s*(?:#.*)?)$"
|
||||
)
|
||||
|
||||
|
||||
@@ -51,57 +66,6 @@ class DockerImageVersionUpdate:
|
||||
latest: str
|
||||
|
||||
|
||||
def _parse_versioned_tag(tag: str) -> tuple[str, str] | None:
|
||||
match = _VERSIONED_TAG_RE.match(str(tag).strip())
|
||||
if match is None:
|
||||
return None
|
||||
return match.group("semver"), match.group("flavor") or ""
|
||||
|
||||
|
||||
def is_semver(value: str) -> bool:
|
||||
return _parse_versioned_tag(value) is not None
|
||||
|
||||
|
||||
def version_key(tag: str) -> tuple[int, ...]:
|
||||
parsed = _parse_versioned_tag(tag)
|
||||
if parsed is None:
|
||||
return (0,) * 4
|
||||
semver, _flavor = parsed
|
||||
parts = tuple(int(part) for part in semver.lstrip("v").split("."))
|
||||
return parts + (0,) * (4 - len(parts))
|
||||
|
||||
|
||||
def version_depth(tag: str) -> int:
|
||||
parsed = _parse_versioned_tag(tag)
|
||||
if parsed is None:
|
||||
return 0
|
||||
semver, _flavor = parsed
|
||||
return len(semver.lstrip("v").split("."))
|
||||
|
||||
|
||||
def version_flavor(tag: str) -> str:
|
||||
"""Return the `-<flavor>` suffix of a versioned tag, or "" when none.
|
||||
|
||||
Only tags that share the same flavor are considered upgrade candidates
|
||||
for one another, so that e.g. `5.4.5-php8.3-apache` is never
|
||||
auto-bumped to `5.4.6-php8.4-apache` (different runtime flavor) or to
|
||||
`5.4.6-php8.3-fpm` (different webserver flavor).
|
||||
"""
|
||||
parsed = _parse_versioned_tag(tag)
|
||||
return parsed[1] if parsed else ""
|
||||
|
||||
|
||||
def latest_semver(tags: list[str], depth: int, flavor: str = "") -> str | None:
|
||||
candidates = [
|
||||
tag
|
||||
for tag in tags
|
||||
if is_semver(tag)
|
||||
and version_depth(tag) == depth
|
||||
and version_flavor(tag) == flavor
|
||||
]
|
||||
return max(candidates, key=version_key, default=None)
|
||||
|
||||
|
||||
def is_dockerhub(image: str) -> bool:
|
||||
parsed = split_registry_and_name(image)
|
||||
if parsed is None:
|
||||
@@ -194,11 +158,11 @@ def fetch_ghcr_tags(image: str) -> list[str]:
|
||||
|
||||
|
||||
def suppressed_services(config_path: Path) -> set[str]:
|
||||
"""Return service names whose `version:` line is annotated with the
|
||||
unified ``# nocheck: docker-version`` marker.
|
||||
"""Return service names whose ``version:`` line is annotated with
|
||||
the unified ``# nocheck: docker-version`` marker.
|
||||
|
||||
The file root of `meta/services.yml` IS the services map. There
|
||||
is no `services.` wrapper to walk into.
|
||||
The file root of ``meta/services.yml`` IS the services map; there
|
||||
is no ``services.`` wrapper to walk into.
|
||||
"""
|
||||
raw = read_text(str(config_path))
|
||||
lines = raw.splitlines()
|
||||
@@ -235,8 +199,6 @@ def collect_entries(repo_root: Path) -> list[DockerImageVersionEntry]:
|
||||
entries: list[DockerImageVersionEntry] = []
|
||||
|
||||
for ref in iter_role_images(repo_root):
|
||||
if not ref.role.startswith("web-"):
|
||||
continue
|
||||
if ref.source_file != ROLE_FILE_META_SERVICES:
|
||||
continue
|
||||
if not is_semver(ref.version):
|
||||
@@ -293,13 +255,13 @@ def find_outdated_updates(repo_root: Path) -> list[DockerImageVersionUpdate]:
|
||||
|
||||
|
||||
def update_config_versions(config_path: Path, service_versions: dict[str, str]) -> bool:
|
||||
"""Rewrite each ``<service>.version:`` value in `meta/services.yml`.
|
||||
"""Rewrite each ``<service>.version:`` value in ``meta/services.yml``.
|
||||
|
||||
The file root IS the services map (no `services.`
|
||||
wrapper), so the walker tracks one nesting level: top-level service keys
|
||||
and their immediate ``version:`` field.
|
||||
The file root IS the services map (no ``services.`` wrapper), so the
|
||||
walker tracks one nesting level: top-level service keys and their
|
||||
immediate ``version:`` field.
|
||||
"""
|
||||
lines = config_path.read_text( # nocheck: cache-read — function reads then rewrites config_path; cached value would go stale on subsequent calls
|
||||
lines = config_path.read_text( # nocheck: cache-read — read-then-write of the same file; cached read would go stale
|
||||
encoding="utf-8"
|
||||
).splitlines(keepends=True)
|
||||
changed = False
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Detect and rewrite outdated git `ref:` values in `roles/*/meta/services.yml`.
|
||||
|
||||
Counterpart to :mod:`utils.update.docker` (which bumps OCI image
|
||||
`version:` tags). Both share the semver primitives in
|
||||
:mod:`utils.update.base`; this module owns the recursive walker over
|
||||
the services map and the upstream-tag lookup via
|
||||
``git ls-remote --tags <repository>``.
|
||||
|
||||
Recursive walker: every dict at any depth (top-level entity,
|
||||
sub-entity, plugin map, ...) that declares BOTH `repository:` and
|
||||
`ref:` is a candidate. Only semver-compatible refs are
|
||||
version-checkable; branches (`master`, `main`, `stable`) and commit
|
||||
SHAs are skipped.
|
||||
|
||||
Each unique repository URL is queried once per run; the result is
|
||||
reused for every entry pointing at the same repo.
|
||||
|
||||
Suppress a check by placing ``# nocheck: repository-version`` on the
|
||||
line directly above the `ref:` key.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from utils.annotations.suppress import is_suppressed_at
|
||||
from utils.cache.files import read_text
|
||||
from utils.cache.yaml import load_yaml_any
|
||||
from utils.roles.mapping import ROLE_FILE_META_SERVICES
|
||||
from utils.update.base import (
|
||||
is_semver,
|
||||
latest_semver,
|
||||
version_depth,
|
||||
version_flavor,
|
||||
version_key,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
_NOCHECK_MARKER = "repository-version"
|
||||
|
||||
_REF_VALUE_RE = re.compile(
|
||||
r"^(?P<prefix>\s*ref\s*:\s*)"
|
||||
r"(?P<quote>[\"']?)(?P<value>[^\"'#\s]+)(?P=quote)"
|
||||
r"(?P<suffix>\s*(?:#.*)?)$"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RepositoryRefEntry:
|
||||
role: str
|
||||
entity_path: tuple[str, ...]
|
||||
repository: str
|
||||
ref: str
|
||||
config_path: Path
|
||||
line: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RepositoryRefUpdate:
|
||||
entry: RepositoryRefEntry
|
||||
latest: str
|
||||
|
||||
|
||||
def walk_repo_ref_pairs(
|
||||
node, path: tuple[str, ...]
|
||||
) -> Iterator[tuple[tuple[str, ...], str, str]]:
|
||||
"""Yield ``(entity_path, repository, ref)`` for every dict in the
|
||||
tree that declares both keys with truthy string values."""
|
||||
if isinstance(node, dict):
|
||||
repo = node.get("repository")
|
||||
ref = node.get("ref")
|
||||
if isinstance(repo, str) and repo and isinstance(ref, str) and ref:
|
||||
yield (path, repo.strip(), ref.strip())
|
||||
for key, value in node.items():
|
||||
if key in ("repository", "ref"):
|
||||
continue
|
||||
yield from walk_repo_ref_pairs(value, (*path, str(key)))
|
||||
elif isinstance(node, list):
|
||||
for index, item in enumerate(node):
|
||||
yield from walk_repo_ref_pairs(item, (*path, f"[{index}]"))
|
||||
|
||||
|
||||
def git_ls_remote_tags(url: str) -> list[str]:
|
||||
"""Return tag names from ``git ls-remote --tags``, stripped of the
|
||||
``refs/tags/`` prefix and the ``^{}`` peeled-tag marker."""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["git", "ls-remote", "--tags", url],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return []
|
||||
if proc.returncode != 0:
|
||||
return []
|
||||
tags: set[str] = set()
|
||||
for line in proc.stdout.splitlines():
|
||||
parts = line.split("\t")
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
name = parts[1]
|
||||
if not name.startswith("refs/tags/"):
|
||||
continue
|
||||
name = name[len("refs/tags/") :]
|
||||
name = name.removesuffix("^{}")
|
||||
if name:
|
||||
tags.add(name)
|
||||
return sorted(tags)
|
||||
|
||||
|
||||
def _ref_lines(lines: list[str]) -> list[tuple[int, str]]:
|
||||
"""Return ``(1-indexed line number, ref value)`` for every `ref:`
|
||||
declaration in *lines*."""
|
||||
out: list[tuple[int, str]] = []
|
||||
for index, line in enumerate(lines, start=1):
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith(("ref:", "ref :")):
|
||||
continue
|
||||
try:
|
||||
_, value = stripped.split(":", 1)
|
||||
except ValueError:
|
||||
continue
|
||||
value = value.split("#", 1)[0].strip().strip("'\"")
|
||||
if value:
|
||||
out.append((index, value))
|
||||
return out
|
||||
|
||||
|
||||
def suppressed_ref_lines(config_path: Path) -> set[int]:
|
||||
"""Return 1-indexed line numbers of `ref:` declarations that carry
|
||||
a ``# nocheck: repository-version`` marker on the preceding line."""
|
||||
raw = read_text(str(config_path))
|
||||
lines = raw.splitlines()
|
||||
suppressed: set[int] = set()
|
||||
for line_no, _ref_value in _ref_lines(lines):
|
||||
if is_suppressed_at(lines, line_no, _NOCHECK_MARKER, mode="line-above"):
|
||||
suppressed.add(line_no)
|
||||
return suppressed
|
||||
|
||||
|
||||
def collect_entries(repo_root: Path) -> list[RepositoryRefEntry]:
|
||||
"""Return every semver-checkable ``(repository, ref)`` declaration
|
||||
under ``roles/*/meta/services.yml`` whose ref is not suppressed."""
|
||||
roles_root = repo_root / "roles"
|
||||
entries: list[RepositoryRefEntry] = []
|
||||
for role_dir in sorted(p for p in roles_root.iterdir() if p.is_dir()):
|
||||
services_path = role_dir / ROLE_FILE_META_SERVICES
|
||||
if not services_path.is_file():
|
||||
continue
|
||||
data = load_yaml_any(str(services_path), default_if_missing=None)
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
|
||||
raw = read_text(str(services_path))
|
||||
# Sequential ref→line cursor: when the same ref value appears
|
||||
# at multiple lines, each candidate consumes the next match so
|
||||
# nested entries with identical refs each get their own line.
|
||||
ref_line_index: dict[str, list[int]] = {}
|
||||
for line_no, ref_value in _ref_lines(raw.splitlines()):
|
||||
ref_line_index.setdefault(ref_value, []).append(line_no)
|
||||
|
||||
suppressed = suppressed_ref_lines(services_path)
|
||||
for entity_path, repo, ref in walk_repo_ref_pairs(data, ()):
|
||||
if not is_semver(ref):
|
||||
continue
|
||||
candidates = ref_line_index.get(ref) or []
|
||||
if not candidates:
|
||||
continue
|
||||
line_no = candidates.pop(0)
|
||||
if line_no in suppressed:
|
||||
continue
|
||||
entries.append(
|
||||
RepositoryRefEntry(
|
||||
role=role_dir.name,
|
||||
entity_path=entity_path,
|
||||
repository=repo,
|
||||
ref=ref,
|
||||
config_path=services_path,
|
||||
line=line_no,
|
||||
)
|
||||
)
|
||||
return entries
|
||||
|
||||
|
||||
def find_outdated_updates(repo_root: Path) -> list[RepositoryRefUpdate]:
|
||||
"""Cross-reference every collected entry against ``git ls-remote
|
||||
--tags`` and return the entries whose ref is older than the latest
|
||||
semver tag with the same depth and flavor."""
|
||||
entries = collect_entries(repo_root)
|
||||
repo_tags: dict[str, list[str]] = {}
|
||||
for entry in entries:
|
||||
if entry.repository in repo_tags:
|
||||
continue
|
||||
repo_tags[entry.repository] = git_ls_remote_tags(entry.repository)
|
||||
|
||||
updates: list[RepositoryRefUpdate] = []
|
||||
for entry in entries:
|
||||
tags = repo_tags.get(entry.repository, [])
|
||||
if not tags:
|
||||
continue
|
||||
latest = latest_semver(
|
||||
tags,
|
||||
version_depth(entry.ref),
|
||||
version_flavor(entry.ref),
|
||||
)
|
||||
if latest and version_key(entry.ref) < version_key(latest):
|
||||
updates.append(RepositoryRefUpdate(entry=entry, latest=latest))
|
||||
return updates
|
||||
|
||||
|
||||
def update_config_refs(config_path: Path, line_to_new_ref: dict[int, str]) -> bool:
|
||||
"""Rewrite specific ``ref:`` lines in *config_path*. *line_to_new_ref*
|
||||
maps the 1-indexed line number to the desired new value."""
|
||||
text = config_path.read_text( # nocheck: cache-read — read-then-write of the same file; cached read would go stale
|
||||
encoding="utf-8"
|
||||
)
|
||||
lines = text.splitlines(keepends=True)
|
||||
changed = False
|
||||
for line_no, new_ref in line_to_new_ref.items():
|
||||
idx = line_no - 1
|
||||
if idx < 0 or idx >= len(lines):
|
||||
continue
|
||||
match = _REF_VALUE_RE.match(lines[idx].rstrip("\n"))
|
||||
if match is None:
|
||||
continue
|
||||
if match.group("value") == new_ref:
|
||||
continue
|
||||
quote = match.group("quote")
|
||||
rebuilt = (
|
||||
f"{match.group('prefix')}{quote}{new_ref}{quote}{match.group('suffix')}"
|
||||
)
|
||||
if lines[idx].endswith("\n"):
|
||||
rebuilt += "\n"
|
||||
lines[idx] = rebuilt
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
config_path.write_text("".join(lines), encoding="utf-8")
|
||||
return changed
|
||||
|
||||
|
||||
def apply_updates(updates: list[RepositoryRefUpdate]) -> list[Path]:
|
||||
"""Apply every update in *updates* to disk. Returns the list of
|
||||
config files that were actually modified."""
|
||||
grouped: dict[Path, dict[int, str]] = {}
|
||||
for update in updates:
|
||||
grouped.setdefault(update.entry.config_path, {})[update.entry.line] = (
|
||||
update.latest
|
||||
)
|
||||
|
||||
changed_paths: list[Path] = []
|
||||
for config_path, line_to_new_ref in sorted(grouped.items()):
|
||||
if update_config_refs(config_path, line_to_new_ref):
|
||||
changed_paths.append(config_path)
|
||||
return changed_paths
|
||||
Reference in new issue
Block a user