From 87bc1be60f9fc5bbde197d6b41aa0e2274f3f2de Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sun, 1 Nov 2026 17:48:09 +0100 Subject: [PATCH] perf(ci): throttle only Docker Hub copies in the daily mirror and raise the limit The daily mirror waited about 70 s before each of its 195 copies (50 images per hour), roughly 3.8 h of pauses, and logged no single rate-limit error. 37 of the copies come from ghcr.io, registry.gitlab.com, quay.io, registry.opencode.de and mcr.microsoft.com, which the Docker Hub pull limit does not cover, and --only-missing also waited before images it then skipped. The throttle now waits only before an actual copy from docker.io, and the schedule allows 150 Docker Hub copies per hour (24 s apart). Part of #1324 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/cron-images-mirror-all.yml | 6 +-- cli/contributing/mirror/sync/__main__.py | 12 ++--- .../cli/contributing/mirror/test_sync.py | 51 +++++++++++++++++++ 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cron-images-mirror-all.yml b/.github/workflows/cron-images-mirror-all.yml index 97199aada5..a09a4ed028 100644 --- a/.github/workflows/cron-images-mirror-all.yml +++ b/.github/workflows/cron-images-mirror-all.yml @@ -22,7 +22,7 @@ on: default: "." type: string images_per_hour: - description: "⏱️ Optional throttle: max images mirrored per hour" + description: "⏱️ Optional throttle: max Docker Hub copies per hour (default 150)" required: false type: string issues: @@ -48,7 +48,7 @@ on: type: string default: "." images_per_hour: - description: "Optional throttle: max images mirrored per hour" + description: "Optional throttle: max Docker Hub copies per hour (default 150)" required: false type: string issues: @@ -79,7 +79,7 @@ jobs: env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - IMAGES_PER_HOUR: ${{ github.event.inputs.images_per_hour || inputs.images_per_hour || '50' }} + IMAGES_PER_HOUR: ${{ github.event.inputs.images_per_hour || inputs.images_per_hour || '150' }} steps: - name: ⬇️ Checkout diff --git a/cli/contributing/mirror/sync/__main__.py b/cli/contributing/mirror/sync/__main__.py index d453732b46..6973670eef 100644 --- a/cli/contributing/mirror/sync/__main__.py +++ b/cli/contributing/mirror/sync/__main__.py @@ -70,7 +70,7 @@ def main() -> int: "--images-per-hour", type=_validate_positive_int, default=None, - help="Optional throttle: max images to mirror per hour (best-effort).", + help="Optional throttle: max Docker Hub copies per hour (best-effort); other registries are not throttled.", ) args = parser.parse_args() @@ -83,11 +83,6 @@ def main() -> int: last_start_ts: float | None = None for img in chain(iter_role_images(repo_root), iter_runner_images(repo_root)): - last_start_ts = _throttle_before_next_copy( - images_per_hour=args.images_per_hour, - last_start_ts=last_start_ts, - ) - total += 1 src = f"docker://{img.source}" dest = f"docker://{provider.image_base(img)}:{img.version}" @@ -101,6 +96,11 @@ def main() -> int: ) continue + if img.registry == "docker.io": + last_start_ts = _throttle_before_next_copy( + images_per_hour=args.images_per_hour, + last_start_ts=last_start_ts, + ) print(f"[mirror] {label}: {src} -> {dest}", flush=True) provider.mirror(img) diff --git a/tests/unit/python/cli/contributing/mirror/test_sync.py b/tests/unit/python/cli/contributing/mirror/test_sync.py index 2f8860eb69..31b8877077 100644 --- a/tests/unit/python/cli/contributing/mirror/test_sync.py +++ b/tests/unit/python/cli/contributing/mirror/test_sync.py @@ -7,6 +7,57 @@ import cli.contributing.mirror.sync.__main__ as sync_main from cli.contributing.mirror.model import ImageRef +class TestThrottle(unittest.TestCase): + def test_only_docker_hub_copies_wait_for_the_throttle(self) -> None: + def image(name: str, registry: str) -> ImageRef: + return ImageRef( + role="r", + service=name, + name=name, + version="1", + source=f"{registry}/{name}:1", + registry=registry, + ) + + images = [ + image("a", "docker.io"), + image("b", "ghcr.io"), + image("c", "quay.io"), + image("d", "docker.io"), + ] + with ( + patch( + "cli.contributing.mirror.sync.__main__.iter_role_images", + return_value=images, + ), + patch( + "cli.contributing.mirror.sync.__main__.iter_runner_images", + return_value=[], + ), + patch.object(sync_main.GHCRProvider, "image_base", return_value="g/x"), + patch.object(sync_main.GHCRProvider, "mirror") as mock_mirror, + patch.object(sync_main.time, "monotonic", return_value=0.0), + patch.object(sync_main.time, "sleep") as mock_sleep, + patch( + "sys.argv", + [ + "mirror-sync", + "--ghcr-namespace", + "acme", + "--ghcr-repository", + "myrepo", + "--images-per-hour", + "150", + ], + ), + ): + result = sync_main.main() + + self.assertEqual(result, 0) + self.assertEqual(mock_mirror.call_count, 4) + self.assertEqual([call.args for call in mock_sleep.call_args_list], [(24.0,)]) + + class TestMirrorSync(unittest.TestCase): def test_only_missing_skips_existing_destination(self) -> None: image = ImageRef(