refactor(images): move role images from defaults/main.yml to meta/services.yml

Consolidates image declarations under a single schema: every role's
`<svc>.{image,version}` lives in `meta/services.yml` alongside compose
service entries. The `image` lookup now reads from disk per role,
`iter_role_images` collapses to a single source, and the mirror chain
(resolver + apply_mirror_overrides) drops the dead `images:` branch.

`test-e2e-playwright` resolves `TEST_E2E_PLAYWRIGHT_IMAGE` directly via
`lookup('image', 'playwright', 'ref')` in vars/main.yml; the runtime
set_fact + override-empty-string dance is gone.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
kevinveenbirkenbachandClaude Opus 4.7 committed 2026-05-10 22:06:32 +02:00
1 parent 14700e0658
commit 576fad84ff
23 files changed
+188 -370

No files matched your search

@@ -167,8 +167,8 @@ def main(argv: list[str] | None = None) -> int:
default=None,
help=(
"Apply docker image mirror overrides to host_vars "
"applications.*.services.* and images_overrides.*.* "
"(image/version). If used without value, defaults to <repo-root>/mirrors.yml. "
"applications.*.services.* (image/version). "
"If used without value, defaults to <repo-root>/mirrors.yml. "
"If a value is provided, it is treated as a path to the mirrors YAML/JSON file."
),
)
@@ -63,10 +63,8 @@ def apply_mirror_overrides(host_vars_file: Path, mirrors_file: Path) -> None:
)
mirrors_apps = mirrors_raw.get("applications", {}) or {}
mirrors_images = mirrors_raw.get("images", {}) or {}
has_applications = isinstance(mirrors_apps, dict) and bool(mirrors_apps)
has_images = isinstance(mirrors_images, dict) and bool(mirrors_images)
if not has_applications and not has_images:
if not has_applications:
return # no-op
yaml_rt = YAML(typ="rt")
@@ -139,36 +137,6 @@ def apply_mirror_overrides(host_vars_file: Path, mirrors_file: Path) -> None:
svc_doc["version"] = version
changed = True
if has_images:
images_overrides_doc = _ensure_ruamel_map(doc, "images_overrides")
for role_id, role_svcs in mirrors_images.items():
if not isinstance(role_svcs, dict):
continue
role_images_doc = _ensure_ruamel_map(images_overrides_doc, str(role_id))
for svc_name, svc_block in role_svcs.items():
if not isinstance(svc_block, dict):
continue
image = svc_block.get("image")
version = svc_block.get("version")
if not isinstance(image, str) or _is_blank(image):
continue
if not isinstance(version, str) or _is_blank(version):
continue
image = image.strip()
version = version.strip()
svc_images_doc = _ensure_ruamel_map(role_images_doc, str(svc_name))
if _is_blank(svc_images_doc.get("image")):
svc_images_doc["image"] = image
changed = True
if _is_blank(svc_images_doc.get("version")):
svc_images_doc["version"] = version
changed = True
if not changed:
return
+2 -13
View File
@@ -7,15 +7,13 @@ from pathlib import Path
from cli.contributing.mirror.providers import GHCRProvider
from utils.cache.yaml import dump_yaml_str
from utils.docker.image.discovery import iter_role_images
from utils.roles.mapping import ROLE_FILE_DEFAULTS_MAIN
def main() -> int:
parser = argparse.ArgumentParser(
description=(
"Emit the mirrors.yml mapping that redirects every role-declared "
"image (applications.*.services.* + images_overrides.*.*) to its "
"GHCR mirror URI."
"image (applications.*.services.*) to its GHCR mirror URI."
),
)
parser.add_argument("--repo-root", default=".")
@@ -27,17 +25,8 @@ def main() -> int:
repo_root = Path(args.repo_root).resolve()
applications: dict = {}
images: dict = {}
for img in iter_role_images(repo_root):
if img.source_file == ROLE_FILE_DEFAULTS_MAIN:
role_images = images.setdefault(img.role, {})
role_images[str(img.service)] = {
"image": provider.image_base(img),
"version": img.version,
}
continue
app = applications.setdefault(img.role, {})
services = app.setdefault("services", {})
services[str(img.service)] = {
@@ -45,7 +34,7 @@ def main() -> int:
"version": img.version,
}
result = {"applications": applications, "images": images}
result = {"applications": applications}
if args.json:
print(json.dumps(result, indent=2))
@@ -30,11 +30,11 @@ Every Playwright-enabled role MUST provide exactly these two files:
`playwright.config.js` MUST remain centralized in `roles/test-e2e-playwright/files/` and `package.json` in `roles/test-e2e-playwright/templates/package.json.j2` (rendered per-deploy into the staging dir). You MUST NOT duplicate or override either per role.
The `@playwright/test` version pinned into the rendered `package.json` comes from `images.playwright.version` in `roles/test-e2e-playwright/defaults/main.yml` via the `image_version` filter. That YAML value is the single source of truth for the Playwright version.
The `@playwright/test` version pinned into the rendered `package.json` comes from `playwright.version` in `roles/test-e2e-playwright/meta/services.yml` via the `image_version` filter. That YAML value is the single source of truth for the Playwright version.
## CI Image Source 🪞
The Playwright container image is declared in `roles/test-e2e-playwright/defaults/main.yml` under `images.playwright` (`image:` + `version:`). CI mirrors that image into GHCR through the standard mirror pipeline before deploy tests run, so Playwright jobs do not depend on direct pulls from MCR during the test phase.
The Playwright container image is declared in `roles/test-e2e-playwright/meta/services.yml` under the `playwright` service entry (`image:` + `version:`). CI mirrors that image into GHCR through the standard mirror pipeline before deploy tests run, so Playwright jobs do not depend on direct pulls from MCR during the test phase.
The `version:` MUST stay pinned to an exact Playwright version and distro pair such as `v1.59.1-noble`. You MUST NOT switch this to a mutable tag such as `latest`, because mutable upstream tags are more vulnerable to CDN propagation delays and make flaky CI failures much harder to reason about.
+4 -15
View File
@@ -40,7 +40,7 @@ Example: `ghcr.io/kevinveenbirkenbach/infinito-nexus-core/mirror/docker.io/nextc
### Resolver 🗺️
`cli/contributing/mirror/resolver/__main__.py` reads all role images and outputs a `mirrors.yml` file with separate sections for compose images and role-local flat images:
`cli/contributing/mirror/resolver/__main__.py` reads all role images and outputs a `mirrors.yml` file under a single `applications.<role>.services.<service>` map:
```yaml
applications:
@@ -49,18 +49,8 @@ applications:
<service>:
image: ghcr.io/{namespace}/{repository}/mirror/{registry}/{name}
version: <tag>
images:
<role>:
<service>:
image: ghcr.io/{namespace}/{repository}/mirror/{registry}/{name}
version: <tag>
```
The split matches the source declaration:
- `meta/services.yml` image refs stay on the existing `applications.*.services.*` path
- `defaults/main.yml` image refs are emitted under `images.*.*`
This file is consumed by the inventory creator to substitute mirror URLs into host variables.
### Sync 🔄
@@ -77,16 +67,15 @@ The mirror workflow runs as stage 8 of the CI pipeline. See [ci.md](../git/pipel
### CI Flow 📋
1. Image discovery scans role declarations in `meta/services.yml` and `defaults/main.yml`.
1. Image discovery scans role declarations in `meta/services.yml`.
2. [images-mirror-missing.yml](../../../../.github/workflows/images-mirror-missing.yml) copies only missing upstream refs into GHCR via `cli.contributing.mirror.sync --only-missing`. Optional Docker Hub credentials reduce source-side rate limits during that sync.
3. Fork PRs cannot publish packages themselves, so their untrusted `pull_request` runs wait in `scripts/meta/wait/mirrors.sh` until a trusted producer run has published the required refs.
4. Inventory generation writes the resulting mirror refs from `mirrors.yml` back into host vars.
5. Deploy and test jobs resolve images from `ghcr.io/{namespace}/{repository}/mirror/...` instead of pulling directly from Docker Hub, MCR, or other upstream registries.
The mirrors file is generated into the inventory directory. The inventory creator applies mirror image overrides to host variables via `cli/administration/inventory/provision/mirror_overrides.py`, which reads both top-level keys from `mirrors.yml` and writes into host vars as follows:
The mirrors file is generated into the inventory directory. The inventory creator applies mirror image overrides to host variables via `cli/administration/inventory/provision/mirror_overrides.py`, which reads `mirrors.yml.applications` and writes `applications.{role}.services.{service}.image` (and `.version`) into host vars subject to the per-service `mirror_policy`.
- `applications.{role}.services.{service}.image`: Populated from `mirrors.yml.applications` and still subject to `mirror_policy`.
- `images_overrides.{role}.{service}.image`: Populated from `mirrors.yml.images` and consumed by `lookup('image', ...)`.
`lookup('image', ...)` consults inventory-side `images_overrides.{role}.{service}` field-wise on top of the role default. Setting that key by hand in inventory remains supported for non-mirror overrides, but the mirror pipeline writes only into `applications.*`.
### Mirror Policy 📋
+11 -43
View File
@@ -3,52 +3,30 @@
This page describes how Docker images are declared, read, overridden, discovered, and mirrored.
For the mirroring pipeline, see [mirror.md](mirror.md).
## Declaration Formats 📋
## Declaration Format 📋
A role MAY declare images in two ways depending on how the image is used.
### Compose Service Format: `meta/services.yml`
Used for images that are started as part of the role's Docker Compose stack.
Every role MUST declare its Docker images in `meta/services.yml`.
The file root IS the services map keyed by `<service-name>` (no `compose:` and no `services:` wrapper):
```yaml
# roles/<role>/meta/services.yml
<service-name>:
image: <image-name> # e.g. nextcloud, quay.io/keycloak/keycloak
version: <tag> # e.g. latest, 31.0.0
image: <image-name> # e.g. nextcloud, quay.io/keycloak/keycloak, mcr.microsoft.com/playwright
version: <tag> # e.g. latest, 31.0.0, v1.58.2-noble
ports:
inter: 8080 # internal container port (per req-009)
# … other compose fields
```
### Flat Image Format: `defaults/main.yml`
Used for images referenced directly in Ansible tasks or templates rather than through Compose (e.g. images pulled and run ad-hoc by a task).
The role name is implicit from the file path `roles/<role>/defaults/main.yml`.
```yaml
# roles/<role>/defaults/main.yml
images:
<service-name>:
image: <image-name> # e.g. mcr.microsoft.com/playwright, postgres
version: <tag> # e.g. v1.58.2-noble
```
- `image` MUST be the image name as it appears in a `docker pull` command, without the tag.
- `version` MUST be the tag.
- Images without an explicit registry prefix are treated as Docker Hub images.
- Role-local image defaults MUST stay in `images`. Inventory-side overrides MUST NOT be written there.
#### Why `defaults/` and not `vars/`? 🤔
The `images:` block holds role-local defaults, not inventory state.
Keeping it in `defaults/main.yml` makes that intent explicit and keeps the role-side declaration separate from inventory-side `images_overrides`.
Using `vars/` would give those defaults higher precedence and blur the separation.
- Compose-managed services and ad-hoc task images both live under the same key on the role's primary or auxiliary service entries.
- Role-local image defaults MUST stay in `meta/services.yml`. Inventory-side overrides MUST NOT be written there.
## Read 📖
Roles MUST use `lookup('image', ...)` instead of direct `images[...]` access.
Roles MUST use `lookup('image', ...)` instead of reading the `meta/services.yml` keys directly.
The lookup MAY infer the current role id from `role_name`.
If `role_name` is not available, the role id MUST be passed explicitly.
@@ -72,8 +50,8 @@ Supported `want` values:
## Override ✏️
Inventory-side mirror and manual overrides MUST be written to `images_overrides.<role>.<service>.{image,version}`.
`lookup('image', ...)` MUST prefer `images_overrides` and fall back field-wise to role-local `images`.
Inventory generation MUST keep `images_overrides` separate from `images` so role defaults remain readable and stable.
`lookup('image', ...)` MUST prefer `images_overrides` and fall back field-wise to the role's `meta/services.yml` entry.
Inventory generation MUST keep `images_overrides` separate from `meta/services.yml` so role defaults remain readable and stable.
Example host-vars override:
@@ -100,23 +78,13 @@ Images from other registries are ignored by the discovery and mirroring tooling.
## Discovery 🔍
`utils/docker/image/discovery.py` scans all roles and yields `ImageRef` objects:
| Source file | Key path | Used for |
|---|---|---|
| `meta/services.yml` | `<svc>.image` + `.version` (file root IS the services map) | Compose-managed services |
| `defaults/main.yml` | `images.<svc>.image` + `.version` | Ad-hoc task images |
`utils/docker/image/discovery.py` scans every `roles/<role>/meta/services.yml` and yields `ImageRef` objects for each top-level service entry that carries both `image` and `version`.
An `ImageRef` carries: `role`, `service`, `name` (without registry), `version`, `source` (full pull ref), `registry`, and `source_file`.
## Mirror Integration 🪞
After discovery, images are mirrored to GHCR and the mirror URLs are injected back into host variables by the inventory creator.
- `meta/services.yml` declarations are resolved back through
`mirrors.yml.applications.<role>.services.<service>`
- `defaults/main.yml` declarations are resolved back through
`mirrors.yml.images.<role>.<service>` and land in host vars under
`images_overrides.<role>.<service>`
Declarations are resolved back through `mirrors.yml.applications.<role>.services.<service>` and land in host vars at the same path.
See [mirror.md](mirror.md) for the full flow.
+24 -6
View File
@@ -1,10 +1,15 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from utils.cache.yaml import load_yaml_any
from utils.roles import PROJECT_ROOT
from utils.roles.mapping import ROLE_FILE_META_SERVICES
_VALID_WANTS = frozenset({"all", "image", "version", "ref"})
@@ -22,8 +27,11 @@ class LookupModule(LookupBase):
- lookup('image', service_name[, want])
- lookup('image', role_id, service_name[, want])
The lookup prefers images_overrides.<role>.<service> over local defaults
from images.<service>, falling back field-wise when an override is absent.
Defaults are sourced from ``roles/<role_id>/meta/services.yml`` under
the matching ``<service_name>`` entry's ``image`` and ``version``
fields. Inventory overrides at
``images_overrides.<role_id>.<service_name>`` win field-wise over the
role default.
"""
def run(
@@ -68,9 +76,7 @@ class LookupModule(LookupBase):
if want not in _VALID_WANTS:
raise AnsibleError("image: want must be one of all, image, version, ref")
defaults = vars_.get("images", {}) or {}
if not isinstance(defaults, dict):
raise AnsibleError("image: Ansible variable 'images' must be a mapping")
defaults = self._load_role_services(role_id)
overrides_root = vars_.get("images_overrides", {}) or {}
if not isinstance(overrides_root, dict):
@@ -114,6 +120,18 @@ class LookupModule(LookupBase):
"image: could not infer role_id; set role_name or pass role_id explicitly"
)
@staticmethod
def _load_role_services(role_id: str) -> dict[str, Any]:
services_path = Path(PROJECT_ROOT) / "roles" / role_id / ROLE_FILE_META_SERVICES
if not services_path.is_file():
raise AnsibleError(f"image: missing {services_path} for role '{role_id}'")
loaded = load_yaml_any(str(services_path), default_if_missing={}) or {}
if not isinstance(loaded, dict):
raise AnsibleError(
f"image: {services_path} must be a YAML mapping at the file root"
)
return loaded
@staticmethod
def _merge_entry(
*,
@@ -127,7 +145,7 @@ class LookupModule(LookupBase):
default_entry = defaults.get(service_name, {})
if default_entry is not None and not isinstance(default_entry, dict):
raise AnsibleError(
f"image: images.{service_name} must be a mapping in role '{role_id}'"
f"image: meta/services.yml entry '{service_name}' must be a mapping in role '{role_id}'"
)
if isinstance(default_entry, dict):
for key in ("image", "version"):
-5
View File
@@ -1,9 +1,4 @@
---
images:
csp-checker:
image: ghcr.io/kevinveenbirkenbach/csp-checker
version: stable
# List of domains whose network block failures (e.g., ORB) should be ignored
# during CSP checks. This is useful for suppressing known external resources
# (e.g., third-party CDNs) that cannot be influenced but otherwise cause
+3
View File
@@ -1,3 +1,6 @@
---
csp:
lifecycle: beta
csp-checker:
image: ghcr.io/kevinveenbirkenbach/csp-checker
version: stable
-13
View File
@@ -1,13 +0,0 @@
---
# ------------------------------------------------------------
# Docker-in-Docker DNS test images
# Auto-mirrored via images discovery; consumers resolve through
# lookup('image', <service>, 'ref').
# ------------------------------------------------------------
images:
busybox:
image: busybox
version: "1.36"
node:
image: node
version: "20-alpine"
@@ -1,3 +1,9 @@
---
svc-container:
lifecycle: beta
busybox:
image: busybox
version: "1.36"
node:
image: node
version: "20-alpine"
+4 -4
View File
@@ -60,14 +60,14 @@ For the file-level contract, use [Contributing `playwright.env.j2`](../../docs/a
`package.json` and `playwright.config.js` are provided centrally by this role:
- `roles/test-e2e-playwright/templates/package.json.j2` (rendered per-deploy; pins `@playwright/test` from `images.playwright.version`)
- `roles/test-e2e-playwright/templates/package.json.j2` (rendered per-deploy; pins `@playwright/test` from `meta/services.yml.playwright.version`)
- `roles/test-e2e-playwright/files/playwright.config.js` (copied as-is)
## Included files
This role ships central Playwright defaults:
- `templates/package.json.j2`: `@playwright/test` version derived from `images.playwright.version`
- `templates/package.json.j2`: `@playwright/test` version derived from `meta/services.yml.playwright.version`
- `files/playwright.config.js`: shared Playwright configuration
Both are used as central defaults for every app role.
@@ -81,7 +81,7 @@ Both are used as central defaults for every app role.
### Playwright runtime
- `TEST_E2E_PLAYWRIGHT_IMAGE` (default: empty; optional full image override. When empty, derived from `images.playwright.image` + `images.playwright.version`)
- `TEST_E2E_PLAYWRIGHT_IMAGE` (resolved in `vars/main.yml` from `meta/services.yml.playwright.image` + `.version` via `lookup('image', 'playwright', 'ref')`)
- `TEST_E2E_PLAYWRIGHT_IMAGE_DISTRO` (default: `noble`)
- `TEST_E2E_PLAYWRIGHT_COMMAND` (default: `npm install --no-fund --no-audit && npx playwright test`)
@@ -99,7 +99,7 @@ Both are used as central defaults for every app role.
## Design notes
- The runner is intentionally test-agnostic at runtime: it executes only tests provided by application roles.
- `images.playwright.version` in `defaults/main.yml` is the single source of truth for the default Playwright version; the central `templates/package.json.j2` pins it via the `image_version` filter.
- `playwright.version` in `meta/services.yml` is the single source of truth for the default Playwright version; the central `templates/package.json.j2` pins it via the `image_version` filter.
- `templates/playwright.env.j2` acts as the stable marker for discovery and as the source of environment configuration.
- Playwright is executed in Docker for reproducibility and consistent browser dependencies.
- In `TLS_MODE=self_signed`, the role requires `CA_TRUST.cert_host`, `CA_TRUST.wrapper_host`, and `CA_TRUST.trust_name` and fails early if cert/wrapper files are missing.
+1 -10
View File
@@ -1,11 +1,5 @@
---
images:
playwright:
image: mcr.microsoft.com/playwright
version: "v1.59.1-noble"
# Derived from images.playwright.version — strip leading "v" and distro suffix.
TEST_E2E_PLAYWRIGHT_VERSION: "{{ images.playwright.version | image_version }}"
TEST_E2E_PLAYWRIGHT_VERSION: "{{ lookup('image', 'playwright', 'version') | image_version }}"
# Where we stage per-role Playwright projects on the host
TEST_E2E_PLAYWRIGHT_STAGE_BASE_DIR: "/tmp/test-e2e-playwright"
@@ -13,9 +7,6 @@ TEST_E2E_PLAYWRIGHT_STAGE_BASE_DIR: "/tmp/test-e2e-playwright"
# Where to store per-role reports/artifacts (host dir)
TEST_E2E_PLAYWRIGHT_REPORTS_BASE_DIR: "/var/lib/infinito/logs/test-e2e-playwright"
# Optional Playwright image override.
TEST_E2E_PLAYWRIGHT_IMAGE: ""
# Command executed inside the Playwright container
TEST_E2E_PLAYWRIGHT_COMMAND: "npm install --no-fund --no-audit && npx playwright test"
@@ -1,3 +1,6 @@
---
test-e2e-playwright:
lifecycle: beta
playwright:
image: mcr.microsoft.com/playwright
version: "v1.59.1-noble"
+2 -11
View File
@@ -110,15 +110,6 @@
msg: "Missing central Playwright package.json template: {{ _playwright_default_package_template }}"
when: not _default_package_stat.stat.exists
- name: Resolve Playwright image
ansible.builtin.set_fact:
_playwright_image: >-
{{
TEST_E2E_PLAYWRIGHT_IMAGE
if (TEST_E2E_PLAYWRIGHT_IMAGE | default('') | trim | length > 0)
else lookup('image', 'playwright', 'ref')
}}
# The `container` wrapper insists the image is present locally before
# `run`. Pull it explicitly so a pruned cache doesn't fail the stage.
- name: Pull Playwright image if missing
@@ -126,7 +117,7 @@
argv:
- container
- pull
- "{{ _playwright_image }}"
- "{{ TEST_E2E_PLAYWRIGHT_IMAGE }}"
register: _playwright_pull
changed_when: false
failed_when: false
@@ -290,7 +281,7 @@
- "{{ _playwright_reports_dir }}:/reports"
- -w
- /e2e
- "{{ _playwright_image }}"
- "{{ TEST_E2E_PLAYWRIGHT_IMAGE }}"
- /bin/bash
- -lc
- "{{ TEST_E2E_PLAYWRIGHT_COMMAND }}"
+2
View File
@@ -0,0 +1,2 @@
---
TEST_E2E_PLAYWRIGHT_IMAGE: "{{ lookup('image', 'playwright', 'ref') }}"
-3
View File
@@ -58,9 +58,6 @@ def _collect_entries() -> list[dict]:
# Only web-* roles
if not ref.role.startswith("web-"):
continue
# Only services from meta/services.yml, not defaults/main.yml.
if ref.source_file != ROLE_FILE_META_SERVICES:
continue
# Only semver versions (pure or `<semver>-<flavor>`)
if not is_semver(ref.version):
continue
@@ -21,8 +21,7 @@ Mirrors the resolution logic of `plugins/lookup/config.py`:
- `users.<canonical>.<sub>`: requires the role's `meta/users.yml` to
declare `<canonical>` AND the global `user_defaults[<canonical>]` to
expose `<sub>`.
- `credentials.<key>` / `images.<key>`: same fallbacks as the
literal-paths check.
- `credentials.<key>`: same fallbacks as the literal-paths check.
- everything else: must walk the role's `application_defaults` entry.
"""
@@ -312,11 +312,11 @@ class TestApplyMirrorOverrides(unittest.TestCase):
self.assertNotIn("web-app-wordpress", out.get("applications", {}))
def test_noop_when_mirrors_has_no_applications_or_images(self) -> None:
def test_noop_when_mirrors_has_no_applications(self) -> None:
host_vars_data = {
"applications": {"a": {"services": {"s": {"image": "i", "version": "v"}}}}
}
mirrors_data = {"not_applications_or_images": {"x": 1}}
mirrors_data = {"not_applications": {"x": 1}}
self._write_yaml(self.host_vars, host_vars_data)
self._write_yaml(self.mirrors, mirrors_data)
@@ -378,77 +378,6 @@ class TestApplyMirrorOverrides(unittest.TestCase):
self.assertNotIn("images_overrides", out)
def test_images_entries_merge_into_images_overrides_only(self) -> None:
host_vars_data = {
"applications": {
"web-app-nextcloud": {
"services": {
"app": {
"image": "docker.io/library/nextcloud",
"version": "30.0.0",
}
}
}
}
}
mirrors_data = {
"images": {
"test-e2e-playwright": {
"playwright": {
"image": "ghcr.io/acme/mirror/mcr.microsoft.com/playwright",
"version": "v1.58.2-noble",
}
}
}
}
self._write_yaml(self.host_vars, host_vars_data)
self._write_yaml(self.mirrors, mirrors_data)
apply_mirror_overrides(self.host_vars, self.mirrors)
out = self._read_yaml(self.host_vars)
svc = out["applications"]["web-app-nextcloud"]["services"]["app"]
self.assertEqual(svc["image"], "docker.io/library/nextcloud")
self.assertEqual(svc["version"], "30.0.0")
img = out["images_overrides"]["test-e2e-playwright"]["playwright"]
self.assertEqual(
img["image"], "ghcr.io/acme/mirror/mcr.microsoft.com/playwright"
)
self.assertEqual(img["version"], "v1.58.2-noble")
def test_images_entries_fill_only_missing_override_fields(self) -> None:
host_vars_data = {
"images_overrides": {
"test-e2e-playwright": {
"playwright": {
"image": "ghcr.io/custom/playwright",
}
}
}
}
mirrors_data = {
"images": {
"test-e2e-playwright": {
"playwright": {
"image": "ghcr.io/acme/mirror/mcr.microsoft.com/playwright",
"version": "v1.58.2-noble",
}
}
}
}
self._write_yaml(self.host_vars, host_vars_data)
self._write_yaml(self.mirrors, mirrors_data)
apply_mirror_overrides(self.host_vars, self.mirrors)
out = self._read_yaml(self.host_vars)
img = out["images_overrides"]["test-e2e-playwright"]["playwright"]
self.assertEqual(img["image"], "ghcr.io/custom/playwright")
self.assertEqual(img["version"], "v1.58.2-noble")
if __name__ == "__main__":
unittest.main()
@@ -7,11 +7,11 @@ from unittest.mock import patch
import cli.contributing.mirror.resolver.__main__ as resolver_main
from cli.contributing.mirror.model import ImageRef
from utils.cache.yaml import load_yaml_str
from utils.roles.mapping import ROLE_FILE_DEFAULTS_MAIN, ROLE_FILE_META_SERVICES
from utils.roles.mapping import ROLE_FILE_META_SERVICES
class TestResolverOutputFormat(unittest.TestCase):
"""resolver emits per-role services map and a separate images key."""
"""resolver emits per-role services map under the applications key."""
def _run_resolver(
self, images: list[ImageRef], extra_argv: list[str] | None = None
@@ -40,7 +40,7 @@ class TestResolverOutputFormat(unittest.TestCase):
self.assertEqual(result, 0)
return load_yaml_str(buf.getvalue()) or {}
def test_compose_images_go_to_applications_key(self) -> None:
def test_meta_services_image_goes_to_applications_key(self) -> None:
image = ImageRef(
role="web-app-nextcloud",
service="app",
@@ -53,13 +53,12 @@ class TestResolverOutputFormat(unittest.TestCase):
out = self._run_resolver([image])
self.assertIn("applications", out)
self.assertIn("images", out)
self.assertNotIn("images", out)
svc = out["applications"]["web-app-nextcloud"]["services"]["app"]
self.assertEqual(svc["version"], "31.0.0")
self.assertIn("ghcr.io/acme/myrepo/mirror/docker.io/nextcloud", svc["image"])
self.assertEqual(out["images"], {})
def test_defaults_images_go_to_images_key(self) -> None:
def test_test_role_image_also_goes_to_applications_key(self) -> None:
image = ImageRef(
role="test-e2e-playwright",
service="playwright",
@@ -67,19 +66,18 @@ class TestResolverOutputFormat(unittest.TestCase):
version="v1.58.2-noble",
source="mcr.microsoft.com/playwright:v1.58.2-noble",
registry="mcr.microsoft.com",
source_file=ROLE_FILE_DEFAULTS_MAIN,
source_file=ROLE_FILE_META_SERVICES,
)
out = self._run_resolver([image])
self.assertEqual(out["applications"], {})
svc = out["images"]["test-e2e-playwright"]["playwright"]
svc = out["applications"]["test-e2e-playwright"]["services"]["playwright"]
self.assertEqual(svc["version"], "v1.58.2-noble")
self.assertIn(
"ghcr.io/acme/myrepo/mirror/mcr.microsoft.com/playwright",
svc["image"],
)
def test_multiple_services_same_role_stay_grouped_per_source_type(self) -> None:
def test_multiple_services_same_role_stay_grouped(self) -> None:
images = [
ImageRef(
role="web-app-nextcloud",
@@ -99,26 +97,15 @@ class TestResolverOutputFormat(unittest.TestCase):
registry="docker.io",
source_file=ROLE_FILE_META_SERVICES,
),
ImageRef(
role="web-app-nextcloud",
service="backup",
name="restic",
version="latest",
source="docker.io/restic/restic:latest",
registry="docker.io",
source_file=ROLE_FILE_DEFAULTS_MAIN,
),
]
out = self._run_resolver(images)
self.assertIn("app", out["applications"]["web-app-nextcloud"]["services"])
self.assertIn("proxy", out["applications"]["web-app-nextcloud"]["services"])
self.assertIn("backup", out["images"]["web-app-nextcloud"])
def test_empty_images_yield_both_empty_top_level_keys(self) -> None:
def test_empty_images_yield_empty_applications(self) -> None:
out = self._run_resolver([])
self.assertEqual(out["applications"], {})
self.assertEqual(out["images"], {})
if __name__ == "__main__": # pragma: no cover
+24 -28
View File
@@ -7,15 +7,15 @@ from unittest.mock import patch
from utils.cache.yaml import load_yaml_str
from utils.docker.image.discovery import iter_role_images
from utils.roles.mapping import ROLE_FILE_DEFAULTS_MAIN, ROLE_FILE_META_SERVICES
from utils.roles.mapping import ROLE_FILE_META_SERVICES
def _make_fs(files: dict[str, str]) -> dict[Path, str]:
return {Path(k): textwrap.dedent(v) for k, v in files.items()}
class TestIterRoleImagesVarsImages(unittest.TestCase):
"""iter_role_images reads defaults/main.yml → images and meta/services.yml → services (any registry)."""
class TestIterRoleImagesMetaServices(unittest.TestCase):
"""iter_role_images reads meta/services.yml → services (any registry)."""
def _run(self, fs: dict[str, str]) -> list:
real_fs = _make_fs(fs)
@@ -37,11 +37,10 @@ class TestIterRoleImagesVarsImages(unittest.TestCase):
def test_mcr_playwright_image(self):
refs = self._run(
{
f"roles/test-e2e-playwright/{ROLE_FILE_DEFAULTS_MAIN}": """
images:
playwright:
image: mcr.microsoft.com/playwright
version: v1.58.2-noble
f"roles/test-e2e-playwright/{ROLE_FILE_META_SERVICES}": """
playwright:
image: mcr.microsoft.com/playwright
version: v1.58.2-noble
""",
}
)
@@ -53,16 +52,15 @@ class TestIterRoleImagesVarsImages(unittest.TestCase):
self.assertEqual(ref.version, "v1.58.2-noble")
self.assertEqual(ref.registry, "mcr.microsoft.com")
self.assertEqual(ref.source, "mcr.microsoft.com/playwright:v1.58.2-noble")
self.assertEqual(ref.source_file, ROLE_FILE_DEFAULTS_MAIN)
self.assertEqual(ref.source_file, ROLE_FILE_META_SERVICES)
def test_ghcr_image_strips_registry_from_name(self):
refs = self._run(
{
f"roles/web-app-matrix/{ROLE_FILE_DEFAULTS_MAIN}": """
images:
matrix-chatgpt-bot:
image: ghcr.io/matrixgpt/matrix-chatgpt-bot
version: latest
f"roles/web-app-matrix/{ROLE_FILE_META_SERVICES}": """
matrix-chatgpt-bot:
image: ghcr.io/matrixgpt/matrix-chatgpt-bot
version: latest
""",
}
)
@@ -71,35 +69,33 @@ class TestIterRoleImagesVarsImages(unittest.TestCase):
self.assertEqual(ref.name, "matrixgpt/matrix-chatgpt-bot")
self.assertEqual(ref.registry, "ghcr.io")
self.assertEqual(ref.source, "ghcr.io/matrixgpt/matrix-chatgpt-bot:latest")
self.assertEqual(ref.source_file, ROLE_FILE_DEFAULTS_MAIN)
self.assertEqual(ref.source_file, ROLE_FILE_META_SERVICES)
def test_missing_image_or_version_is_skipped(self):
refs = self._run(
{
f"roles/some-role/{ROLE_FILE_DEFAULTS_MAIN}": """
images:
no-version:
image: ghcr.io/foo/bar
no-image:
version: "1.0"
f"roles/some-role/{ROLE_FILE_META_SERVICES}": """
no-version:
image: ghcr.io/foo/bar
no-image:
version: "1.0"
""",
}
)
self.assertEqual(refs, [])
def test_no_images_key_yields_nothing(self):
def test_no_image_keys_yields_nothing(self):
refs = self._run(
{
f"roles/some-role/{ROLE_FILE_DEFAULTS_MAIN}": """
some_var: value
f"roles/some-role/{ROLE_FILE_META_SERVICES}": """
some-svc:
lifecycle: beta
""",
}
)
self.assertEqual(refs, [])
def test_config_main_yml_ghcr_image(self):
# Per the file root IS the services map (no compose envelope)
# and source_file reports the new path.
def test_meta_services_ghcr_image(self):
refs = self._run(
{
f"roles/web-app-matrix/{ROLE_FILE_META_SERVICES}": """
@@ -119,5 +115,5 @@ class TestIterRoleImagesVarsImages(unittest.TestCase):
self.assertEqual(ref.source_file, ROLE_FILE_META_SERVICES)
if __name__ == "__main__": # pragma: no cover
if __name__ == "__main__":
unittest.main()
+82 -44
View File
@@ -1,74 +1,96 @@
from __future__ import annotations
import tempfile
import textwrap
import unittest
from pathlib import Path
from unittest.mock import patch
from ansible.errors import AnsibleError
from plugins.lookup import image as image_lookup
from plugins.lookup.image import LookupModule
from utils.roles.mapping import ROLE_FILE_META_SERVICES
class TestImageLookup(unittest.TestCase):
def setUp(self) -> None:
self.lookup = LookupModule()
self._tmp = tempfile.TemporaryDirectory()
self.addCleanup(self._tmp.cleanup)
self.repo_root = Path(self._tmp.name)
patcher = patch.object(image_lookup, "PROJECT_ROOT", self.repo_root)
patcher.start()
self.addCleanup(patcher.stop)
def _write_services(self, role_id: str, content: str) -> None:
path = self.repo_root / "roles" / role_id / ROLE_FILE_META_SERVICES
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(textwrap.dedent(content), encoding="utf-8")
def test_explicit_role_id_returns_default_fields(self) -> None:
variables = {
"images": {
"playwright": {
"image": "mcr.microsoft.com/playwright",
"version": "v1.58.2-noble",
}
}
}
self._write_services(
"test-e2e-playwright",
"""
playwright:
image: mcr.microsoft.com/playwright
version: v1.58.2-noble
""",
)
self.assertEqual(
self.lookup.run(
["test-e2e-playwright", "playwright", "image"],
variables=variables,
variables={},
),
["mcr.microsoft.com/playwright"],
)
self.assertEqual(
self.lookup.run(
["test-e2e-playwright", "playwright", "version"],
variables=variables,
variables={},
),
["v1.58.2-noble"],
)
self.assertEqual(
self.lookup.run(
["test-e2e-playwright", "playwright", "ref"],
variables=variables,
variables={},
),
["mcr.microsoft.com/playwright:v1.58.2-noble"],
)
def test_infers_role_id_from_role_name(self) -> None:
variables = {
"role_name": "sys-ctl-hlth-csp",
"images": {
"csp-checker": {
"image": "ghcr.io/kevinveenbirkenbach/csp-checker",
"version": "stable",
}
},
}
self._write_services(
"sys-ctl-hlth-csp",
"""
csp-checker:
image: ghcr.io/kevinveenbirkenbach/csp-checker
version: stable
""",
)
self.assertEqual(
self.lookup.run(["csp-checker", "ref"], variables=variables),
self.lookup.run(
["csp-checker", "ref"],
variables={"role_name": "sys-ctl-hlth-csp"},
),
["ghcr.io/kevinveenbirkenbach/csp-checker:stable"],
)
def test_role_name_wins_even_if_application_id_is_also_set(self) -> None:
self._write_services(
"web-app-nextcloud",
"""
helper:
image: docker.io/acme/helper
version: 1.0.0
""",
)
variables = {
"role_name": "web-app-nextcloud",
"application_id": "nextcloud",
"images": {
"helper": {
"image": "docker.io/acme/helper",
"version": "1.0.0",
}
},
"images_overrides": {
"web-app-nextcloud": {
"helper": {
@@ -89,14 +111,17 @@ class TestImageLookup(unittest.TestCase):
)
def test_override_wins_fieldwise_over_defaults(self) -> None:
self._write_services(
"test-e2e-playwright",
"""
playwright:
image: mcr.microsoft.com/playwright
version: v1.58.2-noble
""",
)
variables = {
"role_name": "test-e2e-playwright",
"images": {
"playwright": {
"image": "mcr.microsoft.com/playwright",
"version": "v1.58.2-noble",
}
},
"images_overrides": {
"test-e2e-playwright": {
"playwright": {
@@ -117,24 +142,30 @@ class TestImageLookup(unittest.TestCase):
)
def test_missing_mapping_raises(self) -> None:
self._write_services(
"sys-ctl-hlth-csp",
"""
csp:
lifecycle: beta
""",
)
with self.assertRaises(AnsibleError):
self.lookup.run(
["sys-ctl-hlth-csp", "csp-checker", "ref"],
variables={"images": {}, "images_overrides": {}},
variables={"images_overrides": {}},
)
def test_missing_meta_services_file_raises(self) -> None:
with self.assertRaises(AnsibleError):
self.lookup.run(
["nonexistent-role", "svc", "ref"],
variables={},
)
def test_implicit_lookup_requires_role_name_when_only_application_id_exists(
self,
) -> None:
variables = {
"application_id": "web-app-nextcloud",
"images": {
"helper": {
"image": "docker.io/acme/helper",
"version": "1.0.0",
}
},
}
variables = {"application_id": "web-app-nextcloud"}
with self.assertRaises(AnsibleError) as ctx:
self.lookup.run(["helper", "ref"], variables=variables)
@@ -142,11 +173,18 @@ class TestImageLookup(unittest.TestCase):
self.assertIn("set role_name or pass role_id explicitly", str(ctx.exception))
def test_invalid_images_overrides_type_raises(self) -> None:
self._write_services(
"sys-ctl-hlth-csp",
"""
csp-checker:
image: ghcr.io/kevinveenbirkenbach/csp-checker
version: stable
""",
)
with self.assertRaises(AnsibleError):
self.lookup.run(
["sys-ctl-hlth-csp", "csp-checker", "ref"],
variables={
"images": {},
"images_overrides": ["not", "a", "mapping"],
},
)
+3 -41
View File
@@ -4,7 +4,7 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING
from utils.cache.yaml import load_yaml_any as _load_yaml_cached
from utils.roles.mapping import ROLE_FILE_DEFAULTS_MAIN, ROLE_FILE_META_SERVICES
from utils.roles.mapping import ROLE_FILE_META_SERVICES
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -47,9 +47,7 @@ class ImageRef:
registry: str = (
"docker.io" # source registry hostname, e.g. docker.io, quay.io, ghcr.io
)
source_file: str = (
ROLE_FILE_META_SERVICES # ROLE_FILE_META_SERVICES or ROLE_FILE_DEFAULTS_MAIN
)
source_file: str = ROLE_FILE_META_SERVICES
def load_yaml(path: Path) -> dict:
@@ -179,16 +177,12 @@ def iter_role_images(repo_root: Path) -> Iterable[ImageRef]:
"""
Yield all ImageRef entries discovered across all roles in *repo_root*.
Sources:
1. roles/**/meta/services.yml → <entity>.{image,version}
2. roles/**/defaults/main.yml → images.<name>.{image,version}
Source: roles/**/meta/services.yml → <entity>.{image,version}.
See docs/contributing/artefact/image.md for the full format reference.
"""
roles_dir = repo_root / "roles"
# 1. Images from meta/services.yml. The file root IS the services map
# keyed by <entity_name>.
for services_file in roles_dir.glob(f"**/{ROLE_FILE_META_SERVICES}"):
role_name = services_file.parent.parent.name
services = load_yaml(services_file)
@@ -218,35 +212,3 @@ def iter_role_images(repo_root: Path) -> Iterable[ImageRef]:
registry=_detect_registry(image),
source_file=ROLE_FILE_META_SERVICES,
)
# 2. Images from defaults/main.yml → images.<name>.{image,version}
for vars_file in roles_dir.glob(f"**/{ROLE_FILE_DEFAULTS_MAIN}"):
role_name = vars_file.parent.parent.name
data = load_yaml(vars_file)
images = data.get("images", {})
if not isinstance(images, dict):
continue
for service_name, entry in images.items():
if not isinstance(entry, dict):
continue
image = (entry.get("image") or "").strip()
version = (entry.get("version") or "").strip()
if not image or not version:
continue
if not is_mirrorable_image(image):
continue
yield ImageRef(
role=role_name,
service=str(service_name),
name=canonical_image_name(image),
version=version,
source=image_source(image, version),
registry=_detect_registry(image),
source_file=ROLE_FILE_DEFAULTS_MAIN,
)