feat(svc-ai-litellm): warm the chat model during the deploy

A backend pulls its preload model at deploy time but only loads it into
memory on the first inference. That cold load outlives a reverse proxy's
read timeout, so the first consumer request comes back as a 504 while every
later one succeeds.

Send one max_tokens=1 completion from inside the litellm container right
after the per-consumer keys exist, where the container id, master key and
port are already resolved. Five retries cover a gateway that is not
listening yet.

This pairs with the ollama keep_alive setting: the warm-up pays the load
once, keep_alive stops it decaying after five idle minutes.

Note this hardens the deploy: a gateway that cannot serve its configured
model now fails the role instead of surfacing later as a consumer error.
That makes the openrouter/auto fallback with its empty API key loud rather
than silent.

The script ships with a mirrored unit test rather than a nocheck waiver; it
patches urlopen and asserts the URL, method, authorization header, model,
max_tokens and that the timeout exceeds a proxy read timeout.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

roles/svc-ai-litellm/files/python/warm_model.py

@@ -0,0 +1,46 @@
+"""Force the gateway to load its chat model before any consumer prompts it.
+
+A backend pulls its preload model at deploy time but only loads it into memory
+on the first inference. That cold load outlives the reverse proxy's read
+timeout, so the first consumer request comes back as a 504 instead of an answer
+while every later one succeeds. Warming here moves the cost into the deploy.
+
+Environment:
+    LITELLM_MK:    master key the gateway accepts.
+    LITELLM_PORT:  port the gateway listens on inside its own container.
+    LITELLM_MODEL: model name to warm.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import urllib.request
+
+MASTER_KEY = os.environ["LITELLM_MK"]
+PORT = os.environ["LITELLM_PORT"]
+MODEL = os.environ["LITELLM_MODEL"]
+TIMEOUT = 900
+
+payload = json.dumps(
+    {
+        "model": MODEL,
+        "messages": [{"role": "user", "content": "ping"}],
+        "max_tokens": 1,
+    }
+).encode()
+
+request = urllib.request.Request(
+    f"http://127.0.0.1:{PORT}/v1/chat/completions",
+    data=payload,
+    headers={
+        "Authorization": f"Bearer {MASTER_KEY}",
+        "Content-Type": "application/json",
+    },
+    method="POST",
+)
+
+with urllib.request.urlopen(request, timeout=TIMEOUT) as response:  # noqa: S310 fixed internal http origin
+    response.read()
+
+print(f"WARMED {MODEL}")

roles/svc-ai-litellm/tasks/01_keys.yml

@@ -16,3 +16,14 @@
   changed_when: "'CHANGED' in litellm_keys_result.stdout"
   no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
   delegate_to: "{{ LITELLM_HOST_NODE }}"
+
+- name: "🔥 Warm the chat model so the first prompt is not a cold load"
+  ansible.builtin.shell: | # nocheck: container-exec-resolver  runtime-resolved CID, delegated to the hosting node
+    container exec -i -e LITELLM_MK={{ LITELLM_MASTER_KEY | quote }} -e LITELLM_PORT={{ LITELLM_PORT | quote }} -e LITELLM_MODEL={{ LITELLM_CHAT_MODEL | quote }} {{ LITELLM_LOCAL_CID | quote }} python3 -c {{ lookup('file', lookup('path_absolute', 'roles/svc-ai-litellm/files/python/warm_model.py')) | quote }}
+  register: litellm_warm_result
+  retries: 5
+  delay: 10
+  until: litellm_warm_result.rc == 0
+  changed_when: false
+  no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
+  delegate_to: "{{ LITELLM_HOST_NODE }}"

tests/unit/python/roles/svc-ai-litellm/files/test_warm_model.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import importlib.util
+import io
+import json
+import unittest
+from contextlib import redirect_stdout
+from typing import Self
+from unittest.mock import patch
+
+from . import PROJECT_ROOT
+
+SCRIPT_PATH = PROJECT_ROOT / "roles/svc-ai-litellm/files/python/warm_model.py"
+
+ENVIRONMENT = {
+    "LITELLM_MK": "sk-master-key",
+    "LITELLM_PORT": "4000",
+    "LITELLM_MODEL": "smollm2:135m",
+}
+
+
+class FakeResponse:
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, *exc_info: object) -> bool:
+        return False
+
+    def read(self) -> bytes:
+        return b'{"choices": []}'
+
+
+def run_script() -> None:
+    spec = importlib.util.spec_from_file_location("warm_model", SCRIPT_PATH)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+
+class TestWarmModel(unittest.TestCase):
+    def test_posts_a_minimal_completion_to_the_local_gateway(self) -> None:
+        seen: dict[str, object] = {}
+
+        def fake_urlopen(request: object, timeout: float | None = None) -> FakeResponse:
+            seen["url"] = request.full_url
+            seen["method"] = request.get_method()
+            seen["headers"] = {k.lower(): v for k, v in request.header_items()}
+            seen["body"] = json.loads(request.data.decode())
+            seen["timeout"] = timeout
+            return FakeResponse()
+
+        with (
+            patch.dict("os.environ", ENVIRONMENT, clear=False),
+            patch("urllib.request.urlopen", fake_urlopen),
+            redirect_stdout(io.StringIO()) as captured,
+        ):
+            run_script()
+
+        self.assertEqual(seen["url"], "http://127.0.0.1:4000/v1/chat/completions")
+        self.assertEqual(seen["method"], "POST")
+        self.assertEqual(seen["headers"]["authorization"], "Bearer sk-master-key")
+        self.assertEqual(seen["headers"]["content-type"], "application/json")
+        self.assertEqual(seen["body"]["model"], "smollm2:135m")
+        self.assertEqual(seen["body"]["max_tokens"], 1)
+        self.assertIn("WARMED smollm2:135m", captured.getvalue())
+
+    def test_waits_far_longer_than_a_proxy_read_timeout(self) -> None:
+        seen: dict[str, object] = {}
+
+        def fake_urlopen(request: object, timeout: float | None = None) -> FakeResponse:
+            seen["timeout"] = timeout
+            return FakeResponse()
+
+        with (
+            patch.dict("os.environ", ENVIRONMENT, clear=False),
+            patch("urllib.request.urlopen", fake_urlopen),
+            redirect_stdout(io.StringIO()),
+        ):
+            run_script()
+
+        self.assertGreater(seen["timeout"], 60)
+
+
+if __name__ == "__main__":
+    unittest.main()