diff --git a/cli/meta/ci/query.py b/cli/meta/ci/query.py index 127eee8c49..5c47a39cca 100644 --- a/cli/meta/ci/query.py +++ b/cli/meta/ci/query.py @@ -41,7 +41,7 @@ MODES = ("compose", "swarm", "host") def expands_variants(mode: str) -> bool: """Whether *mode* queries, ranks and triggers one CI job per variant. Swarm does (``role#variant`` tokens through the whole chain); compose - and host bundle every variant into whole-role jobs (variant_bundles + and host bundle every variant into whole-role jobs (variant.bundles SPOT), so their query rows stay whole-role too.""" return mode == "swarm" diff --git a/cli/meta/roles/applications/complexity/model.py b/cli/meta/roles/applications/complexity/model.py index c4d045ccef..5cae153b7f 100644 --- a/cli/meta/roles/applications/complexity/model.py +++ b/cli/meta/roles/applications/complexity/model.py @@ -8,7 +8,7 @@ import subprocess from typing import TYPE_CHECKING, Any, NamedTuple from utils.cache.applications import get_variants -from utils.github.variant_bundles import compose_bundle_counts +from utils.github.variant.bundles import compose_bundle_counts from utils.roles.lifecycle import tested_lifecycles from utils.roles.meta_lookup import ( MetaServicesShapeError, diff --git a/docs/contributing/design/variants.md b/docs/contributing/design/variants.md index 8591d2fa1c..32bde60be3 100644 --- a/docs/contributing/design/variants.md +++ b/docs/contributing/design/variants.md @@ -17,7 +17,7 @@ The development deploy CLI uses a **folder-per-round model** that splits cleanly 5. **Cleanup between rounds.** Before each non-first iteration the wrapper runs [entity.sh](../../../scripts/tests/deploy/local/purge/entity.sh) for **every** app in the previous round's include set. The next round therefore boots from an empty host, which guarantees cross-round state coherence: nothing the previous round deployed (whether it stays in the current round's closure or falls out) can leak into round R, so variants are free to disagree on which deps they pull in. The final round is followed by no purge so the last state remains available for inspection or follow-up specs. 6. **`--full-cycle` (async update pass).** When set (or `full_cycle=true` on `make compose-deploy`), the wrapper runs each round's deploy TWICE: first the regular sync pass, then immediately a second pass with `-e ASYNC_ENABLED=true` overriding the host_var, against the SAME variant's folder, BEFORE moving to the next round. The two passes are therefore always co-located on the same host state; the async pass never accidentally targets a host that a previous round's variant left behind. 7. **Pinning rounds (single or bundle).** Both init and deploy accept `--variant ` (same semantics as the `variant=` make arg): a single index (`1`) pins one round, a comma-separated list (`0,1,2`) iterates that subset, and empty/unset is the full matrix. When the selection is a single round no inter-round cleanup runs (no previous round to diff against); out-of-range indices raise a clean error. The `make compose-deploy mode=update` flows honour the same `variant=` arg by suffixing `INFINITO_INVENTORY_DIR` / `INFINITO_INVENTORY_FILE` to `-` (single index), so a redeploy can target one specific variant without iterating the full matrix. -8. **CI runner-split.** The deploy-matrix discovery ([variant_bundles.py](../../../utils/github/variant_bundles.py), wired via [output_apps.sh](../../../scripts/github/resolve/output_apps.sh)) splits any role with more than `INFINITO_VARIANT_BUNDLE_SIZE` (default 3) variants into bundles of consecutive indices — one matrix entry (runner) per bundle, e.g. a 5-variant role becomes `0,1,2` + `3,4`. Each runner receives its slice through the `variant` env (consumed by `--variant`) and runs every variant in its bundle to completion (no time-based cutting). +8. **CI runner-split.** The deploy-matrix discovery ([bundles.py](../../../utils/github/variant/bundles.py), wired via [output_apps.sh](../../../scripts/github/resolve/output_apps.sh)) splits any role with more than `INFINITO_VARIANT_BUNDLE_SIZE` (default 3) variants into bundles of consecutive indices — one matrix entry (runner) per bundle, e.g. a 5-variant role becomes `0,1,2` + `3,4`. Each runner receives its slice through the `variant` env (consumed by `--variant`) and runs every variant in its bundle to completion (no time-based cutting). The in-play loader (`utils.cache.applications.get_merged_applications`) uses each role's variant-free base config (the assembled `meta/.yml` payload) as the default and lets the inventory's `applications.` overrides win via deep merge: the inventory itself is the source of truth for what variant a round runs against. Variant 0 enumerates every dynamic service-key from `meta/services.yml` with `enabled: true, shared: true` so the "all flags on" baseline is explicit and deep-merge with a non-baseline override yields the expected variant-N shape (see [variants.md](../artefact/files/role/variants.md) for the file-format rule and [test_non_baseline_explicit_disables.py](../../../tests/integration/roles/meta/variants/test_non_baseline_explicit_disables.py) for the lint guard). diff --git a/pyproject.toml b/pyproject.toml index f8581d4435..2e07c5097b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -237,7 +237,6 @@ ignore = [ "plugins/lookup/prometheus_integration_active.py" = ["BLE001"] "plugins/lookup/version.py" = ["BLE001"] "utils/cache/base.py" = ["BLE001"] -"utils/github/variant_bundles.py" = ["BLE001"] "utils/nexus/list_apt_proxy_repos.py" = ["BLE001"] "utils/roles/applications/in_group_deps.py" = ["BLE001"] "utils/roles/applications/services/resources.py" = ["BLE001"] diff --git a/scripts/github/resolve/output_apps.sh b/scripts/github/resolve/output_apps.sh index e8b313f638..524ab1ccd2 100755 --- a/scripts/github/resolve/output_apps.sh +++ b/scripts/github/resolve/output_apps.sh @@ -15,7 +15,7 @@ source "${REPO_ROOT}/scripts/meta/env/python.sh" apps="$(./scripts/meta/resolve/apps.sh)" [[ -n "$apps" ]] || apps='[]' -matrix="$(printf '%s' "$apps" | "${PYTHON}" -m utils.github.variant_bundles)" +matrix="$(printf '%s' "$apps" | "${PYTHON}" -m utils.github.variant.bundles)" [[ -n "$matrix" ]] || matrix='[]' echo "apps=$matrix" >>"$GITHUB_OUTPUT" diff --git a/scripts/meta/resolve/apps.sh b/scripts/meta/resolve/apps.sh index 1fdda32082..78d7576715 100755 --- a/scripts/meta/resolve/apps.sh +++ b/scripts/meta/resolve/apps.sh @@ -6,7 +6,7 @@ set -euo pipefail # lifecycle envelope, INFINITO_MAX_JOBS cap with 'auto') lives in # cli.meta.ci.query, shared with the deploy-plan table. Compose and host # emit whole role names; swarm emits per-variant "role#variant" tokens. -# variant_bundles maps the list onto matrix entries. +# variant.bundles maps the list onto matrix entries. # # Inputs via env (defaults live in default.env, the single source of truth): # INFINITO_DEPLOY_MODE compose|swarm|host (required; workflows set it) diff --git a/tests/unit/utils/github/variant/__init__.py b/tests/unit/utils/github/variant/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/utils/github/test_variant_bundles.py b/tests/unit/utils/github/variant/test_bundles.py similarity index 98% rename from tests/unit/utils/github/test_variant_bundles.py rename to tests/unit/utils/github/variant/test_bundles.py index 487580bf3b..6a70911bdf 100644 --- a/tests/unit/utils/github/test_variant_bundles.py +++ b/tests/unit/utils/github/variant/test_bundles.py @@ -1,4 +1,4 @@ -"""Unit tests for :mod:`utils.github.variant_bundles`.""" +"""Unit tests for :mod:`utils.github.variant.bundles`.""" from __future__ import annotations @@ -8,7 +8,7 @@ from unittest.mock import patch from humanfriendly import parse_size -from utils.github import variant_bundles as vb +from utils.github.variant import bundles as vb class TestChunkIndices(unittest.TestCase): diff --git a/utils/env/handlers/infinito/variant_bundle/max_storage.py b/utils/env/handlers/infinito/variant_bundle/max_storage.py index 59f12f8746..09d63b895b 100644 --- a/utils/env/handlers/infinito/variant_bundle/max_storage.py +++ b/utils/env/handlers/infinito/variant_bundle/max_storage.py @@ -1,13 +1,13 @@ """INFINITO_VARIANT_BUNDLE_MAX_STORAGE: cumulative min_storage budget per CI -runner before the discovery (utils.github.variant_bundles) splits a role onto +runner before the discovery (utils.github.variant.bundles) splits a role onto another runner — so storage-heavy variants are not stacked onto one runner that -then runs too long. Defaults to ``variant_bundles.DEFAULT_MAX_STORAGE``.""" +then runs too long. Defaults to ``variant.limits.DEFAULT_MAX_STORAGE``.""" from __future__ import annotations from typing import TYPE_CHECKING -from utils.github.variant_bundle_limits import DEFAULT_MAX_STORAGE +from utils.github.variant.limits import DEFAULT_MAX_STORAGE if TYPE_CHECKING: from utils.env.builder import BuildContext, EnvBuilder @@ -15,7 +15,7 @@ if TYPE_CHECKING: KEY = "INFINITO_VARIANT_BUNDLE_MAX_STORAGE" COMMENT = ( "Cumulative min_storage per CI runner before the deploy-matrix discovery " - "(utils.github.variant_bundles) splits a role across runners." + "(utils.github.variant.bundles) splits a role across runners." ) diff --git a/utils/env/handlers/infinito/variant_bundle/size.py b/utils/env/handlers/infinito/variant_bundle/size.py index 054d28653a..eb8754073f 100644 --- a/utils/env/handlers/infinito/variant_bundle/size.py +++ b/utils/env/handlers/infinito/variant_bundle/size.py @@ -1,12 +1,12 @@ """INFINITO_VARIANT_BUNDLE_SIZE: how many matrix-deploy variants one CI runner -iterates before the discovery (utils.github.variant_bundles) splits a role into +iterates before the discovery (utils.github.variant.bundles) splits a role into another runner. Defaults to 3 everywhere.""" from __future__ import annotations from typing import TYPE_CHECKING -from utils.github.variant_bundle_limits import DEFAULT_BUNDLE_SIZE +from utils.github.variant.limits import DEFAULT_BUNDLE_SIZE if TYPE_CHECKING: from utils.env.builder import BuildContext, EnvBuilder @@ -14,7 +14,7 @@ if TYPE_CHECKING: KEY = "INFINITO_VARIANT_BUNDLE_SIZE" COMMENT = ( "Variants per CI runner before the deploy-matrix discovery " - "(utils.github.variant_bundles) splits a role across runners." + "(utils.github.variant.bundles) splits a role across runners." ) diff --git a/utils/github/variant/__init__.py b/utils/github/variant/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/utils/github/variant_bundles.py b/utils/github/variant/bundles.py similarity index 97% rename from utils/github/variant_bundles.py rename to utils/github/variant/bundles.py index e9d993bbea..abbe040858 100644 --- a/utils/github/variant_bundles.py +++ b/utils/github/variant/bundles.py @@ -31,14 +31,11 @@ import os import sys from typing import TYPE_CHECKING -from humanfriendly import parse_size +from humanfriendly import InvalidSize, parse_size from utils import PROJECT_ROOT from utils.cache.applications import get_variants -from utils.github.variant_bundle_limits import ( - DEFAULT_BUNDLE_SIZE, - DEFAULT_MAX_STORAGE, -) +from utils.github.variant.limits import DEFAULT_BUNDLE_SIZE, DEFAULT_MAX_STORAGE from utils.roles.applications.services.registry import ( build_service_registry_from_applications, load_applications_from_roles_dir, @@ -83,7 +80,7 @@ def resolve_max_storage(raw: str | None = None) -> int | None: return None try: return int(parse_size(value)) - except Exception: + except InvalidSize: raise ValueError( f"INFINITO_VARIANT_BUNDLE_MAX_STORAGE must be a size like {DEFAULT_MAX_STORAGE!r}, " f"got {value!r}" @@ -274,7 +271,7 @@ def main(argv: Sequence[str] | None = None) -> int: apps = json.loads(raw) if raw.strip() else [] if not isinstance(apps, list): raise SystemExit( - f"variant_bundles: expected a JSON array of app ids, got " + f"variant.bundles: expected a JSON array of app ids, got " f"{type(apps).__name__}" ) variants_per_app = get_variants() diff --git a/utils/github/variant_bundle_limits.py b/utils/github/variant/limits.py similarity index 85% rename from utils/github/variant_bundle_limits.py rename to utils/github/variant/limits.py index cba3fe9eec..5668c51eba 100644 --- a/utils/github/variant_bundle_limits.py +++ b/utils/github/variant/limits.py @@ -1,6 +1,6 @@ """Default bundle limits for the CI deploy-matrix discovery. -Kept apart from :mod:`utils.github.variant_bundles`, which reaches yaml and +Kept apart from :mod:`utils.github.variant.bundles`, which reaches yaml and humanfriendly at import time: the env handlers that seed these defaults are imported by the .env generator on the bare bootstrap python, where no third-party module exists. This module must stay stdlib-only.