mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 19:01:54 +00:00
Wrap `docker compose up` in a retry loop (6 attempts, 30s delay) to mitigate temporary Docker Hub / registry 5xx pull failures. Add unit tests covering success-after-retries, exhaustion behavior, and no-sleep on first success. https://chatgpt.com/share/698b8d67-0084-800f-ad33-ec8933dfba15
311 lines
9.8 KiB
Python
311 lines
9.8 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from .coredns import CoreDNSCorefileRenderer
|
|
from .proc import run_streaming
|
|
|
|
|
|
class Compose:
|
|
"""
|
|
Small wrapper around:
|
|
INFINITO_DISTRO=<distro> docker compose --profile ci ...
|
|
"""
|
|
|
|
def __init__(self, repo_root: Path, distro: str) -> None:
|
|
self.repo_root = repo_root
|
|
self.distro = distro
|
|
|
|
def _base_env(self) -> dict[str, str]:
|
|
env = dict(os.environ)
|
|
env["INFINITO_DISTRO"] = self.distro
|
|
return env
|
|
|
|
def _is_ci(self) -> bool:
|
|
# keep this conservative (CI -> no TTY by default)
|
|
return (
|
|
os.environ.get("GITHUB_ACTIONS") == "true"
|
|
or os.environ.get("RUNNING_ON_GITHUB") == "true"
|
|
or os.environ.get("CI") == "true"
|
|
)
|
|
|
|
def run(
|
|
self,
|
|
args: list[str],
|
|
*,
|
|
check: bool = True,
|
|
capture: bool = False,
|
|
live: bool = False,
|
|
keep_lines: int = 400,
|
|
text: bool = True,
|
|
extra_env: dict[str, str] | None = None,
|
|
) -> subprocess.CompletedProcess:
|
|
cmd = ["docker", "compose", "--profile", "ci", *args]
|
|
env = self._base_env()
|
|
if extra_env:
|
|
env.update({k: str(v) for k, v in extra_env.items()})
|
|
|
|
if live:
|
|
r = run_streaming(
|
|
cmd, cwd=self.repo_root, env=env, keep_lines=keep_lines, text=text
|
|
)
|
|
else:
|
|
r = subprocess.run(
|
|
cmd,
|
|
cwd=self.repo_root,
|
|
env=env,
|
|
check=False, # handle check ourselves for consistent behavior
|
|
capture_output=capture,
|
|
text=text,
|
|
)
|
|
|
|
if check and int(r.returncode) != 0:
|
|
raise subprocess.CalledProcessError(
|
|
int(r.returncode), cmd, output=r.stdout, stderr=r.stderr
|
|
)
|
|
|
|
return r
|
|
|
|
def _compose_up_with_retries(
|
|
self,
|
|
args: list[str],
|
|
*,
|
|
attempts: int = 6,
|
|
delay_s: int = 30,
|
|
) -> None:
|
|
"""
|
|
Retry the underlying `docker compose ... up` to mitigate transient registry errors
|
|
(e.g., Docker Hub 500 while pulling coredns).
|
|
"""
|
|
last_exc: Exception | None = None
|
|
|
|
for i in range(1, int(attempts) + 1):
|
|
try:
|
|
self.run(args, check=True)
|
|
return
|
|
except Exception as exc:
|
|
last_exc = exc
|
|
|
|
if i >= int(attempts):
|
|
# Re-raise the last error to keep previous behavior (fail CI).
|
|
raise
|
|
|
|
print(
|
|
f">>> WARNING: compose up failed (attempt {i}/{attempts}): {exc}\n"
|
|
f">>> Retrying in {int(delay_s)}s..."
|
|
)
|
|
time.sleep(int(delay_s))
|
|
|
|
# Should be unreachable, but keep mypy happy.
|
|
if last_exc is not None:
|
|
raise last_exc
|
|
|
|
def _render_coredns_corefile(self) -> None:
|
|
renderer = CoreDNSCorefileRenderer(repo_root=self.repo_root)
|
|
out = renderer.render(show_preview=True, preview_lines=25)
|
|
print(f"[compose] Corefile generated at: {out}")
|
|
print(
|
|
f"[compose] Corefile exists={out.exists()} "
|
|
f"size={out.stat().st_size if out.exists() else 'n/a'}"
|
|
)
|
|
|
|
def up(self, *, run_entry_init: bool = True) -> None:
|
|
print(">>> Rendering CoreDNS Corefile from template")
|
|
self._render_coredns_corefile()
|
|
|
|
print(">>> Starting compose stack (coredns + infinito)")
|
|
env = self._base_env()
|
|
keys = [
|
|
"INFINITO_DISTRO",
|
|
"INFINITO_IMAGE",
|
|
"INFINITO_IMAGE_TAG",
|
|
"INFINITO_PULL_POLICY",
|
|
"INFINITO_NO_BUILD",
|
|
"GITHUB_SHA",
|
|
]
|
|
print(">>> env:", {k: env.get(k) for k in keys})
|
|
print(">>> NIX_CONFIG:", "<set>" if env.get("NIX_CONFIG") else "<empty>")
|
|
|
|
# IMPORTANT: use the same env snapshot that docker compose will use
|
|
no_build = env.get("INFINITO_NO_BUILD", "0") == "1"
|
|
# Compose env-file precedence: later files override earlier ones.
|
|
# So env.local should come AFTER env.ci to override CI defaults locally.
|
|
args = ["--env-file", "env.ci"]
|
|
|
|
env_local = self.repo_root / "env.development"
|
|
if env_local.exists():
|
|
print(f">>> Using local env override: {env_local}")
|
|
args += ["--env-file", "env.development"]
|
|
else:
|
|
print(">>> No env.local found (skipping)")
|
|
|
|
args += ["up", "-d"]
|
|
if no_build:
|
|
args.append("--no-build")
|
|
args += ["coredns", "infinito"]
|
|
|
|
# Retry to avoid transient registry/HTTP 5xx errors when pulling images.
|
|
self._compose_up_with_retries(args, attempts=6, delay_s=30)
|
|
|
|
self.wait_for_healthy()
|
|
|
|
if run_entry_init:
|
|
print(">>> Running infinito entry.sh init")
|
|
r = self.exec(
|
|
["sh", "-lc", "/opt/src/infinito/scripts/docker/entry.sh true"],
|
|
workdir="/opt/src/infinito",
|
|
check=False,
|
|
live=True, # <--- live output
|
|
keep_lines=400, # <--- tail for failure printing
|
|
extra_env={
|
|
"ANSIBLE_FORCE_COLOR": "1",
|
|
"PY_COLORS": "1",
|
|
"TERM": "xterm-256color",
|
|
},
|
|
)
|
|
|
|
if r.returncode != 0:
|
|
print("===== entry.sh stdout (tail) =====")
|
|
print((r.stdout or "").rstrip() or "<empty>")
|
|
print("===== entry.sh stderr (tail) =====")
|
|
print((r.stderr or "").rstrip() or "<empty>")
|
|
raise RuntimeError(f"entry.sh init failed (rc={r.returncode})")
|
|
|
|
def down(self) -> None:
|
|
"""
|
|
Tear down the infinito docker compose stack for this repo/distro.
|
|
"""
|
|
# IMPORTANT:
|
|
# Keep the same behavior as cli.deploy.development.down (volumes + CI cleanup).
|
|
# Local import avoids runtime import cycles.
|
|
from .down import down_stack
|
|
|
|
down_stack(repo_root=self.repo_root, distro=self.distro)
|
|
|
|
def exec(
|
|
self,
|
|
cmd: list[str],
|
|
*,
|
|
check: bool = True,
|
|
workdir: str | None = None,
|
|
capture: bool = False,
|
|
live: bool = False,
|
|
keep_lines: int = 400,
|
|
tty: bool | None = None,
|
|
extra_env: dict[str, str] | None = None,
|
|
) -> subprocess.CompletedProcess:
|
|
"""
|
|
Execute inside infinito container.
|
|
|
|
TTY behavior:
|
|
- tty=None -> auto (local: True, CI: False)
|
|
- tty=True -> allocate TTY (colors, interactive)
|
|
- tty=False -> -T (CI safe)
|
|
|
|
Streaming:
|
|
- live=True streams stdout/stderr to terminal while keeping a tail buffer.
|
|
"""
|
|
if tty is None:
|
|
tty = not self._is_ci()
|
|
|
|
args = ["exec"]
|
|
if not tty:
|
|
args.append("-T")
|
|
|
|
if workdir:
|
|
args += ["-w", workdir]
|
|
|
|
if extra_env:
|
|
for k, v in extra_env.items():
|
|
args += ["-e", f"{k}={v}"]
|
|
|
|
args += ["infinito", *cmd]
|
|
|
|
return self.run(
|
|
args,
|
|
check=check,
|
|
capture=capture,
|
|
live=live,
|
|
keep_lines=keep_lines,
|
|
)
|
|
|
|
def _get_infinito_container_id(self) -> str:
|
|
"""
|
|
Resolve infinito container ID via docker compose.
|
|
Works independent of project name.
|
|
"""
|
|
r = self.run(["ps", "-q", "infinito"], capture=True, check=True)
|
|
cid = (r.stdout or "").strip()
|
|
|
|
if not cid:
|
|
raise RuntimeError(
|
|
"infinito container not found (docker compose ps -q infinito returned empty)"
|
|
)
|
|
|
|
return cid
|
|
|
|
def wait_for_healthy(self, *, timeout_s: int = 200) -> None:
|
|
"""
|
|
Wait until infinito container is marked healthy by Docker.
|
|
On timeout: print last 200 log lines for debugging.
|
|
"""
|
|
print(">>> Waiting for infinito container to become healthy")
|
|
|
|
cid = self._get_infinito_container_id()
|
|
start = time.time()
|
|
|
|
while True:
|
|
r = subprocess.run(
|
|
["docker", "inspect", "-f", "{{.State.Health.Status}}", cid],
|
|
cwd=self.repo_root,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
|
|
status = r.stdout.strip() if r.returncode == 0 else ""
|
|
|
|
if status == "healthy":
|
|
print(">>> infinito container is healthy")
|
|
return
|
|
|
|
if status == "unhealthy":
|
|
print(">>> infinito container is unhealthy")
|
|
|
|
if (time.time() - start) > timeout_s:
|
|
print(
|
|
">>> ERROR: infinito container not healthy, dumping last 200 log lines\n"
|
|
)
|
|
|
|
logs = self.exec(
|
|
["sh", "-lc", "journalctl -n 200 --no-pager || true"],
|
|
check=False,
|
|
capture=True,
|
|
)
|
|
|
|
print("===== journalctl (last 200 lines) =====")
|
|
print(logs.stdout or "<no output>")
|
|
print("======================================\n")
|
|
|
|
docker_logs = subprocess.run(
|
|
["docker", "logs", "--tail", "200", cid],
|
|
cwd=self.repo_root,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
|
|
print("===== docker logs (last 200 lines) =====")
|
|
print(docker_logs.stdout or "<no output>")
|
|
print("=======================================\n")
|
|
|
|
raise RuntimeError(
|
|
f"infinito container not healthy after {timeout_s}s "
|
|
f"(last status: {status})"
|
|
)
|
|
|
|
time.sleep(2)
|