mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 02:54:26 +00:00
The min_storage gate measures every declared pin on each run, and every measurement was an anonymous manifest request to Docker Hub, which meters those per IP. A full run exhausted the pull budget, so the following local deploy could no longer pull at all and the gate itself stopped measuring: 96 of 165 images answered indeterminately and only warned. Read the GHCR mirror first and fall back to the upstream registry only for images it does not carry. The copy is byte-identical, so the measured size is unchanged. Persist measured sizes under the temporary directory; store positive results only, because a cached indeterminate answer would be remembered as fact and the gate would stop measuring for good. Lift the mirror path formula out of the provider into utils so the writer and the reader share one definition, resolving the namespace from GITHUB_REPOSITORY, then remote.pushDefault, then origin. pushDefault has to win: origin may point at an upstream that hosts no mirror. Parse .git/config by hand because configparser rejects the duplicate keys git writes and the test container ships no git binary. Measured over all 165 declared pins: no requests to Docker Hub, 277 to the mirror, 9 to registries it does not carry, and 164 of 165 images measured. Leave manifest_exists on the upstream registry. It answers whether the declared pin is still pullable from where the deploy pulls it, which a mirrored copy cannot attest; it takes the cache only. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
189 lines
5.6 KiB
Python
189 lines
5.6 KiB
Python
from __future__ import annotations
|
|
|
|
import subprocess
|
|
from abc import ABC, abstractmethod
|
|
from typing import TYPE_CHECKING
|
|
|
|
from utils.docker.mirror import mirror_image_base
|
|
|
|
if TYPE_CHECKING:
|
|
import argparse
|
|
|
|
from .model import ImageRef
|
|
|
|
|
|
class RegistryProvider(ABC):
|
|
@abstractmethod
|
|
def image_base(self, image: ImageRef) -> str:
|
|
pass
|
|
|
|
@abstractmethod
|
|
def mirror(self, image: ImageRef) -> None:
|
|
pass
|
|
|
|
@abstractmethod
|
|
def tag_exists(self, image: ImageRef) -> bool:
|
|
pass
|
|
|
|
|
|
class GHCRProvider(RegistryProvider):
|
|
def __init__(self, namespace: str, repository: str, prefix: str = "mirror") -> None:
|
|
self.namespace = namespace.lower()
|
|
self.repository = repository.lower()
|
|
self.prefix = prefix.strip("/")
|
|
|
|
@classmethod
|
|
def add_args(cls, parser: argparse.ArgumentParser) -> None:
|
|
"""Register --ghcr-namespace, --ghcr-repository, --ghcr-prefix on *parser*."""
|
|
parser.add_argument("--ghcr-namespace", required=True)
|
|
parser.add_argument("--ghcr-repository", required=True)
|
|
parser.add_argument("--ghcr-prefix", default="mirror")
|
|
|
|
@classmethod
|
|
def from_args(cls, args: argparse.Namespace) -> GHCRProvider:
|
|
"""Construct a GHCRProvider from parsed CLI args."""
|
|
return cls(args.ghcr_namespace, args.ghcr_repository, args.ghcr_prefix)
|
|
|
|
def image_base(self, image: ImageRef) -> str:
|
|
return mirror_image_base(
|
|
image.registry,
|
|
image.name,
|
|
namespace=self.namespace,
|
|
repository=self.repository,
|
|
prefix=self.prefix,
|
|
)
|
|
|
|
def tag_exists(self, image: ImageRef) -> bool:
|
|
"""
|
|
Return True if the destination tag already exists in GHCR.
|
|
|
|
Uses: skopeo inspect docker://<dest>
|
|
Exit code:
|
|
- 0 => exists
|
|
- !=0 => does not exist OR cannot be accessed (auth/network)
|
|
"""
|
|
dest = f"{self.image_base(image)}:{image.version}"
|
|
r = subprocess.run(
|
|
["skopeo", "inspect", f"docker://{dest}"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return r.returncode == 0
|
|
|
|
def _run_copy(self, *, src: str, dest: str, extra: list[str] | None = None) -> None:
|
|
cmd = [
|
|
"skopeo",
|
|
"copy",
|
|
"--all",
|
|
"--retry-times",
|
|
"5",
|
|
"--dest-precompute-digests",
|
|
]
|
|
if extra:
|
|
cmd += extra
|
|
cmd += [src, f"docker://{dest}"]
|
|
|
|
subprocess.run(
|
|
cmd,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
def _looks_like_blob_reuse_problem(self, e: subprocess.CalledProcessError) -> bool:
|
|
out = (e.stdout or "") + "\n" + (e.stderr or "")
|
|
s = out.lower()
|
|
return (
|
|
"reuse blob" in s
|
|
or "blob mount" in s
|
|
or "mount blob" in s
|
|
or ("failed to mount" in s and ("403" in s or "401" in s))
|
|
or "denied:" in s
|
|
or "unauthorized" in s
|
|
)
|
|
|
|
def mirror(self, image: ImageRef) -> None:
|
|
dest = f"{self.image_base(image)}:{image.version}"
|
|
src = f"docker://{image.source}"
|
|
|
|
try:
|
|
self._run_copy(src=src, dest=dest)
|
|
|
|
except subprocess.CalledProcessError as e:
|
|
output = (e.stdout or "") + (e.stderr or "")
|
|
if output.strip():
|
|
print(output, flush=True)
|
|
|
|
if self._looks_like_blob_reuse_problem(e):
|
|
self._run_copy(
|
|
src=src,
|
|
dest=dest,
|
|
extra=[
|
|
"--dest-compress-format",
|
|
"gzip",
|
|
"--dest-compress-level",
|
|
"1",
|
|
"--dest-force-compress-format",
|
|
],
|
|
)
|
|
return
|
|
|
|
raise
|
|
|
|
|
|
class GiteaProvider(RegistryProvider):
|
|
def __init__(self, registry: str, namespace: str, prefix: str = "mirror") -> None:
|
|
self.registry = registry.rstrip("/")
|
|
self.namespace = namespace
|
|
self.prefix = prefix.strip("/")
|
|
|
|
def image_base(self, image: ImageRef) -> str:
|
|
return f"{self.registry}/{self.namespace}/{self.prefix}/{image.registry}/{image.name}"
|
|
|
|
def tag_exists(self, image: ImageRef) -> bool:
|
|
"""
|
|
Return True if the destination tag already exists in the target registry.
|
|
|
|
Uses: skopeo inspect docker://<dest>
|
|
Exit code:
|
|
- 0 => exists
|
|
- !=0 => does not exist OR cannot be accessed (auth/network)
|
|
"""
|
|
dest = f"{self.image_base(image)}:{image.version}"
|
|
r = subprocess.run(
|
|
["skopeo", "inspect", f"docker://{dest}"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return r.returncode == 0
|
|
|
|
def mirror(self, image: ImageRef) -> None:
|
|
dest = f"{self.image_base(image)}:{image.version}"
|
|
src = f"docker://{image.source}"
|
|
|
|
cmd = [
|
|
"skopeo",
|
|
"copy",
|
|
"--all",
|
|
"--retry-times",
|
|
"5",
|
|
"--dest-precompute-digests",
|
|
src,
|
|
f"docker://{dest}",
|
|
]
|
|
|
|
try:
|
|
subprocess.run(
|
|
cmd,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
except subprocess.CalledProcessError as e:
|
|
output = (e.stdout or "") + (e.stderr or "")
|
|
if output.strip():
|
|
print(output, flush=True)
|
|
raise
|