feat(cache): serve the Argos model host through the package cache

argos-net.com hosts the .argosmodel files the LibreTranslate image pulls
at build time, 70 to 150 MB each and around 100 of them, so every rebuild
refetched several gigabytes from upstream. Route the host through the
existing Nexus raw proxy and the nginx MITM frontend that already covers
pypi, npm, alpine and the GitHub raw hosts.

The argos server block overrides the global proxy_read_timeout. Nexus
emits no byte of an object until it has fetched and stored the whole
thing, so the wait for the response header is the wait for the entire
cold fetch, which outran the global 300s once eight download lanes
competed for the upstream.

INFINITO_CACHE_PACKAGE_FRONTEND_CONF replaces the hardcoded relative bind
for the frontend's upstream map. One cache stack serves every checkout on
the host, so the map follows the primary checkout by default;
INFINITO_CACHE_CONF_SOURCE=worktree points it at the current one to test a
change before it merges. dotenv-force threads that variable through its
env -i allowlist.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Makefile

@@ -306,7 +306,9 @@ dotenv:
 # Note: avoids stale BASH_ENV INFINITO_* values pinning via setdefault.
 dotenv-force:
 	@rm -f .env
-	@env -i HOME="$${HOME}" PATH="$${PATH}" python3 -m cli.meta.env
+	@env -i HOME="$${HOME}" PATH="$${PATH}" \
+		INFINITO_CACHE_CONF_SOURCE="$${INFINITO_CACHE_CONF_SOURCE:-}" \
+		python3 -m cli.meta.env
 
 .PHONY: environment-bootstrap
 # Bootstrap the local development environment.

compose.cache-consumer.yml

@@ -21,6 +21,7 @@ services:
       - "dl-cdn.alpinelinux.org:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
       - "raw.githubusercontent.com:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
       - "codeload.github.com:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
+      - "argos-net.com:${INFINITO_CACHE_PACKAGE_FRONTEND_IP}"
     volumes:
       - type: bind
         source: ${INFINITO_CACHE_REGISTRY_CA_HOST_PATH:?Run 'make dotenv' to generate the .env single source of truth}
@@ -32,6 +33,7 @@ services:
       - ./scripts/docker/cache/registry-ca.sh:/usr/local/bin/registry-cache-ca.sh:ro
       - ${INFINITO_CACHE_PACKAGE_PIP_CONF:?Set by 'cli deploy development up'}:/etc/pip.conf:ro
       - ${INFINITO_CACHE_PACKAGE_NPMRC:?Set by 'cli deploy development up'}:/root/.npmrc:ro
-      - ./compose/package-cache/apt/${INFINITO_DISTRO:?Run 'make dotenv' to generate the .env single source of truth}.list:/etc/apt/sources.list.d/package-cache.list:ro
+      - ./compose/package-cache/apt/${INFINITO_DISTRO:?Run 'make dotenv' to generate the .env single source of
+        truth}.list:/etc/apt/sources.list.d/package-cache.list:ro
       - ${INFINITO_CACHE_PACKAGE_FRONTEND_CA_FILE:?Set by 'cli deploy development up'}:/opt/package-frontend-ca.crt:ro
       - ./scripts/docker/cache/package-frontend/ca.sh:/usr/local/bin/package-frontend-ca.sh:ro

compose/cache.override.yml

@@ -87,7 +87,7 @@ services:
         read_only: true
         bind:
           create_host_path: true
-      - ./compose/package-cache-frontend/upstreams.conf:/etc/nginx/conf.d/upstreams.conf:ro
+      - ${INFINITO_CACHE_PACKAGE_FRONTEND_CONF:?Run 'make dotenv' to generate the .env single source of truth}:/etc/nginx/conf.d/upstreams.conf:ro
     networks:
       default:
         ipv4_address: ${INFINITO_CACHE_PACKAGE_FRONTEND_IP:?Run 'make dotenv' to generate the .env single source of truth}

compose/package-cache-frontend/cert-gen.sh

@@ -10,7 +10,7 @@ CA_KEY=/ca/ca.key
 CA_DAYS=3650
 LEAF_DAYS=730
 
-HOSTNAMES="pypi.org files.pythonhosted.org registry.npmjs.org rubygems.org index.rubygems.org repo.packagist.org proxy.golang.org dl-cdn.alpinelinux.org raw.githubusercontent.com codeload.github.com"
+HOSTNAMES="pypi.org files.pythonhosted.org registry.npmjs.org rubygems.org index.rubygems.org repo.packagist.org proxy.golang.org dl-cdn.alpinelinux.org raw.githubusercontent.com codeload.github.com argos-net.com"
 
 log() { printf '[package-frontend-certs] %s\n' "$*" >&2; }
 

compose/package-cache-frontend/upstreams.conf

@@ -195,3 +195,19 @@ server {
 		proxy_pass http://package-cache:8081/repository/raw-codeload-github/;
 	}
 }
+
+# Nexus emits no byte of an Argos model until it has fetched and stored the
+# whole 70-150 MB object, so the wait for the response header is the wait for
+# the entire cold fetch. Measured: 46-88s uncontended, and `504 ups=504
+# upstime=300.101` once eight lanes compete. The global 300s stays where the
+# apt mirror failover needs it.
+server {
+	listen 443 ssl;
+	server_name argos-net.com;
+	ssl_certificate     /etc/nginx/certs/argos-net.com.crt;
+	ssl_certificate_key /etc/nginx/certs/argos-net.com.key;
+	location / {
+		proxy_read_timeout 1800s;
+		proxy_pass http://package-cache:8081/repository/raw-argos-net/;
+	}
+}

default.env

@@ -106,6 +106,9 @@ INFINITO_CACHE_REGISTRY_IP=172.30.0.6
 # Bind-mounted registry-cache proxy configuration consumed by the cache container.
 INFINITO_CACHE_REGISTRY_PROXY_CONF=./compose/registry-cache/proxy.conf
 
+# Checkout whose nginx upstream map the shared cache frontend mounts: primary | worktree.
+INFINITO_CACHE_CONF_SOURCE=primary
+
 # True where a persistent pull-through cache stack pays off (local dev).
 INFINITO_CACHE_STACK=true
 

scripts/docker/cache/package.sh

@@ -152,6 +152,7 @@ ensure_all_proxies() {
 	ensure_proxy_repo helm helm-bitnami "$(printf '{"name":"helm-bitnami","online":true,%s,"proxy":{"remoteUrl":"https://charts.bitnami.com/bitnami","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
 	ensure_proxy_repo raw raw-githubusercontent "$(printf '{"name":"raw-githubusercontent","online":true,%s,"proxy":{"remoteUrl":"https://raw.githubusercontent.com/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")"
 	ensure_proxy_repo raw raw-codeload-github "$(printf '{"name":"raw-codeload-github","online":true,%s,"proxy":{"remoteUrl":"https://codeload.github.com/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")"
+	ensure_proxy_repo raw raw-argos-net "$(printf '{"name":"raw-argos-net","online":true,%s,"proxy":{"remoteUrl":"https://argos-net.com/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"raw":{"contentDisposition":"ATTACHMENT"}}' "${storage}")"
 	ensure_proxy_repo rubygems gem-proxy "$(printf '{"name":"gem-proxy","online":true,%s,"proxy":{"remoteUrl":"https://rubygems.org/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
 	ensure_proxy_repo go go-proxy "$(printf '{"name":"go-proxy","online":true,%s,"proxy":{"remoteUrl":"https://proxy.golang.org/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'}}' "${storage}")"
 	ensure_proxy_repo yum yum-rocky "$(printf '{"name":"yum-rocky","online":true,%s,"proxy":{"remoteUrl":"https://download.rockylinux.org/pub/rocky/","contentMaxAge":'"${CACHE_CONTENT_MAX_AGE_MIN}"',"metadataMaxAge":'"${CACHE_METADATA_MAX_AGE_MIN}"'},"yum":{"repodataDepth":5}}' "${storage}")"

tests/unit/python/utils/env/handlers/infinito/test_cache_conf.py

@@ -0,0 +1,99 @@
+"""Unit tests for :mod:`utils.env.handlers.infinito.cache_conf`."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import unittest
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest.mock import patch
+
+from utils.env.builder import BuildContext, EnvBuilder
+from utils.env.handlers.infinito import cache_conf as handler
+
+_BLANK = {handler.SOURCE_KEY: "", handler.KEY: ""}
+
+
+def _ctx(root: Path, static: dict[str, str] | None = None) -> BuildContext:
+    return BuildContext(
+        static=static or {},
+        static_comments={handler.KEY: handler.COMMENT},
+        repo_root=root,
+        on_gha=False,
+        on_act=False,
+    )
+
+
+def _apply(ctx: BuildContext) -> str:
+    eb = EnvBuilder()
+    handler.apply(eb, ctx)
+    return eb.values[handler.KEY]
+
+
+class TestPrimaryCheckout(unittest.TestCase):
+    def setUp(self) -> None:
+        self._tmp = TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        self.root = Path(self._tmp.name).resolve()
+        self.primary = self.root / "primary"
+        self.primary.mkdir()
+        self._git("init", "-q", "-b", "main", cwd=self.primary)
+        self._git("config", "user.email", "t@t", cwd=self.primary)
+        self._git("config", "user.name", "t", cwd=self.primary)
+        (self.primary / "seed").write_text("seed\n")
+        self._git("add", "seed", cwd=self.primary)
+        self._git("commit", "-qm", "seed", cwd=self.primary)
+
+    def _git(self, *args: str, cwd: Path) -> None:
+        subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True)
+
+    def test_the_primary_checkout_resolves_to_itself(self) -> None:
+        self.assertEqual(handler.primary_checkout(self.primary), self.primary)
+
+    def test_a_worktree_resolves_to_the_primary_checkout(self) -> None:
+        linked = self.root / "linked"
+        self._git("worktree", "add", "-q", str(linked), "-b", "side", cwd=self.primary)
+
+        self.assertEqual(handler.primary_checkout(linked), self.primary)
+
+    def test_a_directory_outside_git_falls_back_to_itself(self) -> None:
+        outside = self.root / "outside"
+        outside.mkdir()
+
+        self.assertEqual(handler.primary_checkout(outside), outside)
+
+
+class TestApply(unittest.TestCase):
+    def setUp(self) -> None:
+        self._tmp = TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        self.root = Path(self._tmp.name).resolve()
+
+    @patch.dict(os.environ, _BLANK, clear=False)
+    def test_the_default_follows_the_primary_checkout(self) -> None:
+        with patch.object(handler, "primary_checkout", return_value=Path("/primary")):
+            self.assertEqual(
+                _apply(_ctx(self.root)), str(Path("/primary") / handler.RELATIVE)
+            )
+
+    @patch.dict(os.environ, {**_BLANK, handler.SOURCE_KEY: "worktree"}, clear=False)
+    def test_the_environment_can_pin_the_worktree(self) -> None:
+        self.assertEqual(_apply(_ctx(self.root)), str(self.root / handler.RELATIVE))
+
+    @patch.dict(os.environ, _BLANK, clear=False)
+    def test_a_static_entry_can_pin_the_worktree(self) -> None:
+        ctx = _ctx(self.root, {handler.SOURCE_KEY: "Worktree"})
+
+        self.assertEqual(_apply(ctx), str(self.root / handler.RELATIVE))
+
+    @patch.dict(os.environ, {**_BLANK, handler.SOURCE_KEY: "primary"}, clear=False)
+    def test_an_unknown_source_keeps_the_primary_checkout(self) -> None:
+        with patch.object(handler, "primary_checkout", return_value=Path("/primary")):
+            self.assertEqual(
+                _apply(_ctx(self.root)), str(Path("/primary") / handler.RELATIVE)
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

utils/env/handlers/__init__.py

@@ -16,6 +16,7 @@ from . import (
     passthrough,
 )
 from .infinito import (
+    cache_conf,
     cache_stack,
     container,
     distros,
@@ -23,6 +24,7 @@ from .infinito import (
     domain,
     gpu_count,
     i18n_libretranslate,
+    i18n_tuning,
     image,
     image_repository,
     inventory,
@@ -76,6 +78,7 @@ ORDERED_HANDLERS = [
     domain,
     running_on_act,
     running_on_github,
+    cache_conf,
     cache_stack,
     tor_socks_port,
     is_wsl2,
@@ -97,6 +100,7 @@ ORDERED_HANDLERS = [
     registry_cache_max_size,
     gpu_count,
     i18n_libretranslate,
+    i18n_tuning,
     package_cache_heap,
     package_cache_direct_mem,
     package_cache_blobstore_max,

utils/env/handlers/infinito/cache_conf.py

@@ -0,0 +1,51 @@
+"""INFINITO_CACHE_PACKAGE_FRONTEND_CONF: nginx upstream map the shared
+package-cache frontend mounts, taken from the primary checkout unless
+INFINITO_CACHE_CONF_SOURCE says worktree."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from utils.env.builder import BuildContext, EnvBuilder
+
+KEY = "INFINITO_CACHE_PACKAGE_FRONTEND_CONF"
+SOURCE_KEY = "INFINITO_CACHE_CONF_SOURCE"
+COMMENT = (
+    "Bind source for the package-cache frontend's nginx upstream map. "
+    "One cache stack serves every checkout on the host, so it follows the "
+    f"primary one; set {SOURCE_KEY}=worktree to test a change before merging."
+)
+RELATIVE = Path("compose") / "package-cache-frontend" / "upstreams.conf"
+
+
+def primary_checkout(repo_root: Path) -> Path:
+    """Return the checkout that owns the repository, given any worktree.
+
+    Args:
+        repo_root: the checkout this build runs in.
+
+    Returns:
+        The primary checkout's path, or ``repo_root`` when git cannot say.
+    """
+    common = subprocess.run(
+        ["git", "rev-parse", "--path-format=absolute", "--git-common-dir"],
+        cwd=repo_root,
+        capture_output=True,
+        text=True,
+        check=False,
+    ).stdout.strip()
+    if not common:
+        return repo_root
+    return Path(common).parent
+
+
+def apply(eb: EnvBuilder, ctx: BuildContext) -> None:
+    source = (
+        (os.environ.get(SOURCE_KEY) or ctx.static.get(SOURCE_KEY) or "").strip().lower()
+    )
+    root = ctx.repo_root if source == "worktree" else primary_checkout(ctx.repo_root)
+    eb.setdefault(KEY, str(root / RELATIVE), comment=COMMENT)