kevinveenbirkenbach/computer-playbook
1
0 Fork 0
mirror of https://github.com/kevinveenbirkenbach/computer-playbook.git synced 2026-11-06 10:59:59 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
e6f552ca0adb5cc04acfb83bb17ee2e4e59cf2b5
computer-playbook/scripts/docker/cache/package.sh
T
kevinveenbirkenbachandClaude Opus 5 8d8701d9b3 feat(apt): one Ubuntu mirror list for the package cache and image builds, with failover, a short timeout and a health test 
The ubuntu CI image build (job 103277644636) hung on Canonical's
archive: apt fetched 26 MB at 417 kB/s, then 61.6 kB in 29 s, then sat
on the package download until the job was cancelled after 23 minutes,
while download.docker.com delivered 42.8 MB in under a second on the
same runner. Locally archive.ubuntu.com and security.ubuntu.com took
20-26 s to the first byte against 2 s for a university mirror.

INFINITO_APT_UBUNTU_MIRRORS in default.env is now the one list of
Ubuntu mirrors, in failover order: de.archive.ubuntu.com, ftp.fau.de,
ftp.halifax.rwth-aachen.de, archive.ubuntu.com.

- Image builds: the Dockerfile sources the list alongside the install
  scripts and runs scripts/install/apt-mirrors.sh first. In an image
  with ubuntu.sources it points both stanzas at apt's mirror+file
  method, priorities in list order, and sets Acquire::http::Timeout to
  10 s, so apt moves on from a mirror that fails or goes silent; other
  distros are untouched. In ubuntu:latest apt-get update took 8 s
  against 110 s on the stock sources, a dead first mirror was left
  behind within 25 s against 62 s on apt's defaults, and the swarm node
  packages (80 debs) came in 17 s. The Dockerfile's own RUN lines ran
  clean on ubuntu, debian, fedora and arch.
- Package cache: package.sh proxies the first entry as the primary
  Ubuntu repos and the second as their fallback. Nexus revalidates
  metadata on every request here (metadataMaxAge 0), which costs a
  full download from a mirror without an ETag; both de.archive and
  ftp.fau.de send one, and the 15 MB universe index now comes back
  from the cache in 0.02 s.
- tests/external/repository/test_apt_ubuntu_mirrors.py keeps the list
  honest: the first mirror must answer InRelease within 5 s (median of
  three) or the test names a faster later one, no mirror's -updates may
  lag the freshest by more than 24 h, and every entry must be the
  primary archive, a Canonical country alias or an enabled official
  mirror in Launchpad's registry. It caught the first draft of the
  list: mirrors.edge.kernel.org is not an official Ubuntu mirror.

After pulling, an existing checkout needs `make dotenv-force`: plain
`make dotenv` re-reads the old .env through BASH_ENV and keeps its
values.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-11 17:13:54 +02:00

207 lines
12 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# shellcheck shell=bash
# Idempotent Nexus 3 OSS proxy bootstrap.
# See docs/contributing/environment/cache.md.
set -euo pipefail
: "${INFINITO_CACHE_PACKAGE_HOST_PATH:?Source scripts/meta/env/load.sh first}"
: "${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD:?Source scripts/meta/env/load.sh first}"
: "${INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX:?Source scripts/meta/env/load.sh first}"
: "${INFINITO_CACHE_PACKAGE_MAX_AGE_MIN:?Source scripts/meta/env/load.sh first}"
: "${INFINITO_CACHE_PACKAGE_METADATA_MAX_AGE_MIN:?Source scripts/meta/env/load.sh first}"
: "${INFINITO_APT_UBUNTU_MIRRORS:?Source scripts/meta/env/load.sh first}"
CACHE_CONTENT_MAX_AGE_MIN="${INFINITO_CACHE_PACKAGE_MAX_AGE_MIN}"
CACHE_METADATA_MAX_AGE_MIN="${INFINITO_CACHE_PACKAGE_METADATA_MAX_AGE_MIN}"
read -r -a UBUNTU_MIRRORS <<<"${INFINITO_APT_UBUNTU_MIRRORS}"
UBUNTU_PRIMARY="${UBUNTU_MIRRORS[0]%/}"
UBUNTU_FALLBACK="${UBUNTU_MIRRORS[1]:?INFINITO_APT_UBUNTU_MIRRORS needs a primary and a fallback mirror}"
UBUNTU_FALLBACK="${UBUNTU_FALLBACK%/}"
PKGCACHE_CONTAINER="infinito-package-cache"
NEXUS_REST="http://127.0.0.1:8081/service/rest"
BOOTSTRAP_DONE_FILE="/nexus-data/.infinito-bootstrap-done"
NEXUS_ADMIN_PW_FILE="/nexus-data/.infinito-admin-password"
NEXUS_DEFAULT_PASSWORD="admin123"
log() { printf '[package-cache-bootstrap] %s\n' "$*" >&2; }
nexus_curl() { docker exec "${PKGCACHE_CONTAINER}" curl "$@"; }
wait_for_nexus() {
local _attempt
for _attempt in $(seq 1 120); do
if nexus_curl -fsS "${NEXUS_REST}/v1/status" >/dev/null 2>&1; then
return 0
fi
sleep 2
done
log "Nexus REST not reachable at ${NEXUS_REST}/v1/status after 4 minutes"
return 1
}
store_admin_password() {
printf %s "${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}" |
docker exec -i "${PKGCACHE_CONTAINER}" sh -c "umask 077; cat > '${NEXUS_ADMIN_PW_FILE}'"
}
rotate_admin_password() {
ADMIN_USER="admin"
if docker exec "${PKGCACHE_CONTAINER}" test -f "${BOOTSTRAP_DONE_FILE}"; then
log "Already bootstrapped; using stored admin password"
ADMIN_PASS="$(docker exec "${PKGCACHE_CONTAINER}" cat "${NEXUS_ADMIN_PW_FILE}" 2>/dev/null || true)"
if [ -z "${ADMIN_PASS}" ]; then
ADMIN_PASS="${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}"
fi
return 0
fi
log "Rotating admin password from default '${NEXUS_DEFAULT_PASSWORD}'"
if ! nexus_curl -fsS -u "admin:${NEXUS_DEFAULT_PASSWORD}" \
-H "Content-Type: text/plain" \
-X PUT "${NEXUS_REST}/v1/security/users/admin/change-password" \
--data-binary "${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}"; then
if nexus_curl -fsS -o /dev/null -u "admin:${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}" \
"${NEXUS_REST}/v1/repositories" >/dev/null 2>&1; then
log "Default rotation rejected but configured password authenticates; assuming previous bootstrap, marking done"
else
log "Cannot authenticate with default '${NEXUS_DEFAULT_PASSWORD}' nor with configured INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD."
log "Wipe ${INFINITO_CACHE_PACKAGE_HOST_PATH} (e.g. via 'make clean-cache') and retry."
return 1
fi
fi
store_admin_password
docker exec "${PKGCACHE_CONTAINER}" touch "${BOOTSTRAP_DONE_FILE}"
ADMIN_PASS="${INFINITO_CACHE_PACKAGE_ADMIN_PASSWORD}"
}
ensure_blobstore() {
local quota_gb="${INFINITO_CACHE_PACKAGE_BLOBSTORE_MAX%g}"
local quota_mb=$((quota_gb * 1024))
local payload
payload="$(printf '{"name":"default","path":"default","softQuota":{"type":"spaceUsedQuota","limit":%d}}' "${quota_mb}")"
local code
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
-u "${ADMIN_USER}:${ADMIN_PASS}" \
-H "Content-Type: application/json" \
-X POST "${NEXUS_REST}/v1/blobstores/file" \
--data "${payload}" || true)"
case "${code}" in
201 | 204) log "blobstore default created (HTTP ${code})" ;;
400 | 409) log "blobstore default already exists (HTTP ${code})" ;;
*)
log "unexpected HTTP ${code} creating blobstore"
return 1
;;
esac
}
ensure_repo() {
local format="$1" kind="$2" name="$3" payload="$4"
local code
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
-u "${ADMIN_USER}:${ADMIN_PASS}" \
-H "Content-Type: application/json" \
-X POST "${NEXUS_REST}/v1/repositories/${format}/${kind}" \
--data "${payload}" || true)"
case "${code}" in
201 | 204) log "${format} ${kind} ${name} created (HTTP ${code})" ;;
400 | 409)
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
-u "${ADMIN_USER}:${ADMIN_PASS}" \
-H "Content-Type: application/json" \
-X PUT "${NEXUS_REST}/v1/repositories/${format}/${kind}/${name}" \
--data "${payload}" || true)"
case "${code}" in
200 | 204) log "${format} ${kind} ${name} updated (HTTP ${code})" ;;
*)
log "unexpected HTTP ${code} updating ${format} ${kind} ${name}"
return 1
;;
esac
;;
*)
log "unexpected HTTP ${code} creating ${format} ${kind} ${name}"
return 1
;;
esac
}
ensure_proxy_repo() { ensure_repo "$1" proxy "$2" "$3"; }
# autoBlock stays off: once Nexus blocks a remote it answers every client
# with `404 Remote Auto Blocked` for the whole block window, so a single
# upstream hiccup fails every concurrent job instead of just the request
# that hit it.
ensure_all_proxies() {
local storage='"storage":{"blobStoreName":"default","strictContentTypeValidation":true},"proxy":{"contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"negativeCache":{"enabled":true,"timeToLive":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"httpClient":{"blocked":false,"autoBlock":false}'
ensure_proxy_repo apt apt-debian "$(printf '{"name":"apt-debian","online":true,%s,"proxy":{"remoteUrl":"http://deb.debian.org/debian","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"bookworm","flat":false}}' "${storage}")"
ensure_proxy_repo apt apt-ubuntu "$(printf '{"name":"apt-ubuntu","online":true,%s,"proxy":{"remoteUrl":"'"${UBUNTU_PRIMARY}"'","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"jammy","flat":false}}' "${storage}")"
ensure_proxy_repo apt apt-debian-security "$(printf '{"name":"apt-debian-security","online":true,%s,"proxy":{"remoteUrl":"http://deb.debian.org/debian-security","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"bookworm-security","flat":false}}' "${storage}")" # nocheck: url (bare GET on the deb.debian.org redirector returns 404; apt-client paths underneath dists/bookworm-security/ resolve correctly)
ensure_proxy_repo apt apt-ubuntu-security "$(printf '{"name":"apt-ubuntu-security","online":true,%s,"proxy":{"remoteUrl":"'"${UBUNTU_PRIMARY}"'","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"jammy-security","flat":false}}' "${storage}")"
ensure_proxy_repo apt apt-debian-mirror "$(printf '{"name":"apt-debian-mirror","online":true,%s,"proxy":{"remoteUrl":"http://ftp.debian.org/debian","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"bookworm","flat":false}}' "${storage}")"
ensure_proxy_repo apt apt-ubuntu-mirror "$(printf '{"name":"apt-ubuntu-mirror","online":true,%s,"proxy":{"remoteUrl":"'"${UBUNTU_FALLBACK}"'","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"jammy","flat":false}}' "${storage}")"
ensure_proxy_repo apt apt-debian-security-mirror "$(printf '{"name":"apt-debian-security-mirror","online":true,%s,"proxy":{"remoteUrl":"http://security.debian.org/debian-security","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"bookworm-security","flat":false}}' "${storage}")"
ensure_proxy_repo apt apt-ubuntu-security-mirror "$(printf '{"name":"apt-ubuntu-security-mirror","online":true,%s,"proxy":{"remoteUrl":"'"${UBUNTU_FALLBACK}"'","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"apt":{"distribution":"jammy-security","flat":false}}' "${storage}")"
ensure_proxy_repo pypi pypi-proxy "$(printf '{"name":"pypi-proxy","online":true,%s,"proxy":{"remoteUrl":"https://pypi.org/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
ensure_proxy_repo npm npm-proxy "$(printf '{"name":"npm-proxy","online":true,%s,"proxy":{"remoteUrl":"https://registry.npmjs.org/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
ensure_proxy_repo helm helm-bitnami "$(printf '{"name":"helm-bitnami","online":true,%s,"proxy":{"remoteUrl":"https://charts.bitnami.com/bitnami","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
ensure_proxy_repo raw raw-githubusercontent "$(printf '{"name":"raw-githubusercontent","online":true,%s,"proxy":{"remoteUrl":"https://raw.githubusercontent.com/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")"
ensure_proxy_repo raw raw-codeload-github "$(printf '{"name":"raw-codeload-github","online":true,%s,"proxy":{"remoteUrl":"https://codeload.github.com/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")"
ensure_proxy_repo rubygems gem-proxy "$(printf '{"name":"gem-proxy","online":true,%s,"proxy":{"remoteUrl":"https://rubygems.org/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
ensure_proxy_repo go go-proxy "$(printf '{"name":"go-proxy","online":true,%s,"proxy":{"remoteUrl":"https://proxy.golang.org/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
ensure_proxy_repo yum yum-rocky "$(printf '{"name":"yum-rocky","online":true,%s,"proxy":{"remoteUrl":"https://download.rockylinux.org/pub/rocky/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"yum":{"repodataDepth":5}}' "${storage}")"
ensure_proxy_repo yum yum-fedora "$(printf '{"name":"yum-fedora","online":true,%s,"proxy":{"remoteUrl":"https://download.fedoraproject.org/pub/fedora/linux/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"yum":{"repodataDepth":5}}' "${storage}")"
ensure_proxy_repo raw raw-packagist "$(printf '{"name":"raw-packagist","online":true,%s,"proxy":{"remoteUrl":"https://repo.packagist.org/","contentMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")" # nocheck: url
ensure_proxy_repo raw raw-alpine "$(printf '{"name":"raw-alpine","online":true,%s,"proxy":{"remoteUrl":"https://dl-cdn.alpinelinux.org/alpine/","contentMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")"
}
ensure_eula_accepted() {
local code
# shellcheck disable=SC1112
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
-u "${ADMIN_USER}:${ADMIN_PASS}" \
-H "Content-Type: application/json" \
-X POST "${NEXUS_REST}/v1/system/eula" \
--data '{"accepted":true,"disclaimer":"Use of Sonatype Nexus Repository - Community Edition is governed by the End User License Agreement at https://links.sonatype.com/products/nxrm/ce-eula. By returning the value from ‘accepted:false’ to ‘accepted:true’, you acknowledge that you have read and agree to the End User License Agreement at https://links.sonatype.com/products/nxrm/ce-eula."}' || true)"
case "${code}" in
200 | 204) log "EULA accepted (HTTP ${code})" ;;
*)
log "unexpected HTTP ${code} accepting EULA"
return 1
;;
esac
}
ensure_anonymous_access() {
local code
code="$(nexus_curl -sS -o /dev/null -w '%{http_code}' \
-u "${ADMIN_USER}:${ADMIN_PASS}" \
-H "Content-Type: application/json" \
-X PUT "${NEXUS_REST}/v1/security/anonymous" \
--data '{"enabled":true,"userId":"anonymous","realmName":"NexusAuthorizingRealm"}' || true)"
case "${code}" in
200 | 204) log "anonymous access enabled (HTTP ${code})" ;;
*)
log "unexpected HTTP ${code} enabling anonymous access"
return 1
;;
esac
}
main() {
wait_for_nexus
rotate_admin_password
ensure_eula_accepted
ensure_anonymous_access
ensure_blobstore
ensure_all_proxies
log "bootstrap done"
}
main "$@"
Reference in new issue View Git Blame Copy Permalink