diff --git a/inventories/development/tools.yml.j2 b/inventories/development/tools.yml.j2 index d8df933297..fa231e4418 100644 --- a/inventories/development/tools.yml.j2 +++ b/inventories/development/tools.yml.j2 @@ -40,6 +40,18 @@ applications: services: ollama: cpus: "{{ lookup('resource', 'host_cpus') }}" + min_storage: 4600MB + preload_models: + - alias: qwen2.5:3b + context: 32768 + name: qwen2.5-3b-instruct + repo: lmstudio-community/Qwen2.5-3B-Instruct-GGUF + url: https://huggingface.co/lmstudio-community/Qwen2.5-3B-Instruct-GGUF/resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf + source: https://huggingface.co/lmstudio-community/Qwen2.5-3B-Instruct-GGUF + quant: q4_k_m + file: Qwen2.5-3B-Instruct-Q4_K_M.gguf + sha256: "d44e2c5d1ec3cae1d5cf6a744bee528e46c65a1e66e741fa92730967e7d625bb" + bytes: 1929903008 svc-ai-s1: secrets: credentials: diff --git a/tests/unit/python/utils/inventory/test_tools_model.py b/tests/unit/python/utils/inventory/test_tools_model.py new file mode 100644 index 0000000000..ddc2cdd30c --- /dev/null +++ b/tests/unit/python/utils/inventory/test_tools_model.py @@ -0,0 +1,36 @@ +"""The alias the readiness probe expects is the one the lane deploys.""" + +from __future__ import annotations + +import unittest + +import yaml + +from utils import PROJECT_ROOT +from utils.inventory.tools import LANE_MODEL_ALIAS, render + +OLLAMA_ROLE = "svc-ai-ollama" + + +class TestLaneModel(unittest.TestCase): + def test_the_rendered_inventory_pins_the_alias_the_probe_expects(self) -> None: + """Two places name the model, and nothing else keeps them equal. + + The inventory carries the entry because that is where inventory data + belongs, and the probe needs the bare alias before any render has + run, because ``ensure`` asks whether the lane serves before the + deploy that writes the rendered file. A lane serving one model while + the probe expects another never reports ready and redeploys on every + call. + """ + rendered = yaml.safe_load(render(PROJECT_ROOT, ["en"]).read_text("utf-8")) + ollama = rendered["applications"][OLLAMA_ROLE]["services"]["ollama"] + + self.assertEqual( + [LANE_MODEL_ALIAS], + [model["alias"] for model in ollama["preload_models"]], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/inventory/tools.py b/utils/inventory/tools.py index ae38df1e97..0d53c144ba 100644 --- a/utils/inventory/tools.py +++ b/utils/inventory/tools.py @@ -30,6 +30,14 @@ template already carries. Never reuse it anywhere a real caller could reach. and under the same standing. A deployment hands each consumer a virtual key instead, which is what the role's secret description asks for; the oracle judge is not a deployed consumer and has no vault to read one from. + +The lane pins its own model in the template, rather than taking the +repository's ``AI_LOCAL_MODELS`` default: the oracle needs a judge that can +tell a faithful rendering from an unrelated text, a demand no deployment +makes. :data:`LANE_MODEL_ALIAS` repeats that one name here because the +readiness probe compares it against what the gateway actually serves, and +the rendered inventory is written by the deploy this probe runs before. The +two are held together by a test that renders the template. """ from __future__ import annotations @@ -54,7 +62,7 @@ LITELLM_ROLE = "svc-ai-litellm" OLLAMA_ROLE = "svc-ai-ollama" GATEWAY_ROLE = "web-svc-translate" LANE_SERVICE = "tools" -LOCAL_MODELS_FILE = Path("group_vars") / "all" / "16_ai.yml" +LANE_MODEL_ALIAS = "qwen2.5:3b" PROBE_TIMEOUT_SECONDS = 10 @@ -232,7 +240,7 @@ def _litellm_serves_the_named_model(root: Path) -> bool: """ import json - alias = _named_model(root) + alias = _named_model() body = _answers(f"{host_url(root, LITELLM_ROLE, 'litellm')}/models", LITELLM_KEY) if body is None or not alias: return False @@ -243,15 +251,9 @@ def _litellm_serves_the_named_model(root: Path) -> bool: return alias in served -def _named_model(root: Path) -> str: - """Return the alias the lane's model server is expected to serve. - - Args: - root: repository root. - """ - models = load_yaml(root / LOCAL_MODELS_FILE).get("AI_LOCAL_MODELS") or [] - first = models[0] if models else {} - return str(first.get("alias") or "") if isinstance(first, dict) else "" +def _named_model() -> str: + """Return the alias the lane's model server is expected to serve.""" + return LANE_MODEL_ALIAS def _gateway_covers_the_catalogs(root: Path) -> bool: