From ee9cbc8d5bc54fc41af8bc8ca07f0a803bb3c87e Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sun, 18 Oct 2026 10:30:22 +0200 Subject: [PATCH] feat(tools): pin the lane's judge model in its own inventory The oracle asks a chat model whether a translation states its source. llama3.2:1b cannot answer that: on the suite's own fixture it scored an unrelated text at the middle level instead of the bottom one, and asked as a yes-or-no question it called two identical texts different. With it the suite reached no verdict at all, reporting that the judge read none of the 47 languages the catalogs hold machine translations for. Qwen2.5-3B answers both correctly, and the same run then judged 248 samples. The pin lives in the lane inventory rather than in AI_LOCAL_MODELS, because a judge that can separate a faithful rendering from an unrelated text is a demand no deployment makes, and the default should not grow a gigabyte to satisfy a check. The litellm config builds its routes from the merged application config, so a lane override reaches the model list and its window without touching the repository default. LANE_MODEL_ALIAS repeats the one name in utils/inventory/tools.py, because ensure() asks whether the lane already serves before the deploy that writes the rendered inventory, so the probe cannot read it from there. A lane serving one model while the probe expects another never reports ready and redeploys on every call, so a test renders the template and holds the two names equal. Co-Authored-By: Claude Opus 5 (1M context) --- inventories/development/tools.yml.j2 | 12 +++++++ .../utils/inventory/test_tools_model.py | 36 +++++++++++++++++++ utils/inventory/tools.py | 24 +++++++------ 3 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 tests/unit/python/utils/inventory/test_tools_model.py diff --git a/inventories/development/tools.yml.j2 b/inventories/development/tools.yml.j2 index d8df933297..fa231e4418 100644 --- a/inventories/development/tools.yml.j2 +++ b/inventories/development/tools.yml.j2 @@ -40,6 +40,18 @@ applications: services: ollama: cpus: "{{ lookup('resource', 'host_cpus') }}" + min_storage: 4600MB + preload_models: + - alias: qwen2.5:3b + context: 32768 + name: qwen2.5-3b-instruct + repo: lmstudio-community/Qwen2.5-3B-Instruct-GGUF + url: https://huggingface.co/lmstudio-community/Qwen2.5-3B-Instruct-GGUF/resolve/main/Qwen2.5-3B-Instruct-Q4_K_M.gguf + source: https://huggingface.co/lmstudio-community/Qwen2.5-3B-Instruct-GGUF + quant: q4_k_m + file: Qwen2.5-3B-Instruct-Q4_K_M.gguf + sha256: "d44e2c5d1ec3cae1d5cf6a744bee528e46c65a1e66e741fa92730967e7d625bb" + bytes: 1929903008 svc-ai-s1: secrets: credentials: diff --git a/tests/unit/python/utils/inventory/test_tools_model.py b/tests/unit/python/utils/inventory/test_tools_model.py new file mode 100644 index 0000000000..ddc2cdd30c --- /dev/null +++ b/tests/unit/python/utils/inventory/test_tools_model.py @@ -0,0 +1,36 @@ +"""The alias the readiness probe expects is the one the lane deploys.""" + +from __future__ import annotations + +import unittest + +import yaml + +from utils import PROJECT_ROOT +from utils.inventory.tools import LANE_MODEL_ALIAS, render + +OLLAMA_ROLE = "svc-ai-ollama" + + +class TestLaneModel(unittest.TestCase): + def test_the_rendered_inventory_pins_the_alias_the_probe_expects(self) -> None: + """Two places name the model, and nothing else keeps them equal. + + The inventory carries the entry because that is where inventory data + belongs, and the probe needs the bare alias before any render has + run, because ``ensure`` asks whether the lane serves before the + deploy that writes the rendered file. A lane serving one model while + the probe expects another never reports ready and redeploys on every + call. + """ + rendered = yaml.safe_load(render(PROJECT_ROOT, ["en"]).read_text("utf-8")) + ollama = rendered["applications"][OLLAMA_ROLE]["services"]["ollama"] + + self.assertEqual( + [LANE_MODEL_ALIAS], + [model["alias"] for model in ollama["preload_models"]], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/inventory/tools.py b/utils/inventory/tools.py index ae38df1e97..0d53c144ba 100644 --- a/utils/inventory/tools.py +++ b/utils/inventory/tools.py @@ -30,6 +30,14 @@ template already carries. Never reuse it anywhere a real caller could reach. and under the same standing. A deployment hands each consumer a virtual key instead, which is what the role's secret description asks for; the oracle judge is not a deployed consumer and has no vault to read one from. + +The lane pins its own model in the template, rather than taking the +repository's ``AI_LOCAL_MODELS`` default: the oracle needs a judge that can +tell a faithful rendering from an unrelated text, a demand no deployment +makes. :data:`LANE_MODEL_ALIAS` repeats that one name here because the +readiness probe compares it against what the gateway actually serves, and +the rendered inventory is written by the deploy this probe runs before. The +two are held together by a test that renders the template. """ from __future__ import annotations @@ -54,7 +62,7 @@ LITELLM_ROLE = "svc-ai-litellm" OLLAMA_ROLE = "svc-ai-ollama" GATEWAY_ROLE = "web-svc-translate" LANE_SERVICE = "tools" -LOCAL_MODELS_FILE = Path("group_vars") / "all" / "16_ai.yml" +LANE_MODEL_ALIAS = "qwen2.5:3b" PROBE_TIMEOUT_SECONDS = 10 @@ -232,7 +240,7 @@ def _litellm_serves_the_named_model(root: Path) -> bool: """ import json - alias = _named_model(root) + alias = _named_model() body = _answers(f"{host_url(root, LITELLM_ROLE, 'litellm')}/models", LITELLM_KEY) if body is None or not alias: return False @@ -243,15 +251,9 @@ def _litellm_serves_the_named_model(root: Path) -> bool: return alias in served -def _named_model(root: Path) -> str: - """Return the alias the lane's model server is expected to serve. - - Args: - root: repository root. - """ - models = load_yaml(root / LOCAL_MODELS_FILE).get("AI_LOCAL_MODELS") or [] - first = models[0] if models else {} - return str(first.get("alias") or "") if isinstance(first, dict) else "" +def _named_model() -> str: + """Return the alias the lane's model server is expected to serve.""" + return LANE_MODEL_ALIAS def _gateway_covers_the_catalogs(root: Path) -> bool: