Files
computer-playbook/cli/contributing/mirror/providers.py
kevinveenbirkenbachandClaude Opus 5 f4a713d827 perf(registry): probe the image mirror before Docker Hub and cache sizes
The min_storage gate measures every declared pin on each run, and every measurement was an anonymous manifest request to Docker Hub, which meters those per IP. A full run exhausted the pull budget, so the following local deploy could no longer pull at all and the gate itself stopped measuring: 96 of 165 images answered indeterminately and only warned.

Read the GHCR mirror first and fall back to the upstream registry only for images it does not carry. The copy is byte-identical, so the measured size is unchanged. Persist measured sizes under the temporary directory; store positive results only, because a cached indeterminate answer would be remembered as fact and the gate would stop measuring for good.

Lift the mirror path formula out of the provider into utils so the writer and the reader share one definition, resolving the namespace from GITHUB_REPOSITORY, then remote.pushDefault, then origin. pushDefault has to win: origin may point at an upstream that hosts no mirror. Parse .git/config by hand because configparser rejects the duplicate keys git writes and the test container ships no git binary.

Measured over all 165 declared pins: no requests to Docker Hub, 277 to the mirror, 9 to registries it does not carry, and 164 of 165 images measured.

Leave manifest_exists on the upstream registry. It answers whether the declared pin is still pullable from where the deploy pulls it, which a mirrored copy cannot attest; it takes the cache only.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-30 17:22:29 +02:00

189 lines
5.6 KiB
Python

from __future__ import annotations
import subprocess
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
from utils.docker.mirror import mirror_image_base
if TYPE_CHECKING:
import argparse
from .model import ImageRef
class RegistryProvider(ABC):
@abstractmethod
def image_base(self, image: ImageRef) -> str:
pass
@abstractmethod
def mirror(self, image: ImageRef) -> None:
pass
@abstractmethod
def tag_exists(self, image: ImageRef) -> bool:
pass
class GHCRProvider(RegistryProvider):
def __init__(self, namespace: str, repository: str, prefix: str = "mirror") -> None:
self.namespace = namespace.lower()
self.repository = repository.lower()
self.prefix = prefix.strip("/")
@classmethod
def add_args(cls, parser: argparse.ArgumentParser) -> None:
"""Register --ghcr-namespace, --ghcr-repository, --ghcr-prefix on *parser*."""
parser.add_argument("--ghcr-namespace", required=True)
parser.add_argument("--ghcr-repository", required=True)
parser.add_argument("--ghcr-prefix", default="mirror")
@classmethod
def from_args(cls, args: argparse.Namespace) -> GHCRProvider:
"""Construct a GHCRProvider from parsed CLI args."""
return cls(args.ghcr_namespace, args.ghcr_repository, args.ghcr_prefix)
def image_base(self, image: ImageRef) -> str:
return mirror_image_base(
image.registry,
image.name,
namespace=self.namespace,
repository=self.repository,
prefix=self.prefix,
)
def tag_exists(self, image: ImageRef) -> bool:
"""
Return True if the destination tag already exists in GHCR.
Uses: skopeo inspect docker://<dest>
Exit code:
- 0 => exists
- !=0 => does not exist OR cannot be accessed (auth/network)
"""
dest = f"{self.image_base(image)}:{image.version}"
r = subprocess.run(
["skopeo", "inspect", f"docker://{dest}"],
check=False,
capture_output=True,
text=True,
)
return r.returncode == 0
def _run_copy(self, *, src: str, dest: str, extra: list[str] | None = None) -> None:
cmd = [
"skopeo",
"copy",
"--all",
"--retry-times",
"5",
"--dest-precompute-digests",
]
if extra:
cmd += extra
cmd += [src, f"docker://{dest}"]
subprocess.run(
cmd,
check=True,
capture_output=True,
text=True,
)
def _looks_like_blob_reuse_problem(self, e: subprocess.CalledProcessError) -> bool:
out = (e.stdout or "") + "\n" + (e.stderr or "")
s = out.lower()
return (
"reuse blob" in s
or "blob mount" in s
or "mount blob" in s
or ("failed to mount" in s and ("403" in s or "401" in s))
or "denied:" in s
or "unauthorized" in s
)
def mirror(self, image: ImageRef) -> None:
dest = f"{self.image_base(image)}:{image.version}"
src = f"docker://{image.source}"
try:
self._run_copy(src=src, dest=dest)
except subprocess.CalledProcessError as e:
output = (e.stdout or "") + (e.stderr or "")
if output.strip():
print(output, flush=True)
if self._looks_like_blob_reuse_problem(e):
self._run_copy(
src=src,
dest=dest,
extra=[
"--dest-compress-format",
"gzip",
"--dest-compress-level",
"1",
"--dest-force-compress-format",
],
)
return
raise
class GiteaProvider(RegistryProvider):
def __init__(self, registry: str, namespace: str, prefix: str = "mirror") -> None:
self.registry = registry.rstrip("/")
self.namespace = namespace
self.prefix = prefix.strip("/")
def image_base(self, image: ImageRef) -> str:
return f"{self.registry}/{self.namespace}/{self.prefix}/{image.registry}/{image.name}"
def tag_exists(self, image: ImageRef) -> bool:
"""
Return True if the destination tag already exists in the target registry.
Uses: skopeo inspect docker://<dest>
Exit code:
- 0 => exists
- !=0 => does not exist OR cannot be accessed (auth/network)
"""
dest = f"{self.image_base(image)}:{image.version}"
r = subprocess.run(
["skopeo", "inspect", f"docker://{dest}"],
check=False,
capture_output=True,
text=True,
)
return r.returncode == 0
def mirror(self, image: ImageRef) -> None:
dest = f"{self.image_base(image)}:{image.version}"
src = f"docker://{image.source}"
cmd = [
"skopeo",
"copy",
"--all",
"--retry-times",
"5",
"--dest-precompute-digests",
src,
f"docker://{dest}",
]
try:
subprocess.run(
cmd,
check=True,
capture_output=True,
text=True,
)
except subprocess.CalledProcessError as e:
output = (e.stdout or "") + (e.stderr or "")
if output.strip():
print(output, flush=True)
raise