mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-08 11:46:35 +00:00
kevinveenbirkenbachandClaude Opus 5
a25f475969
--include filters the generated inventory to the ids it is given literally, so a provider pulled in through the dependency closure runs without joining group_names. svc-ai-agent-broker's README therefore documented a deploy that stops on its own assert: litellm comes along, nothing it can serve does, AI_AGENT_MODEL resolves to '' and 00_core.yml refuses the agents. Run 36064589578 shows it as 'Including only application_ids: svc-ai-agent-broker' followed by the assert.
meta/services.yml gains guide_companions, read through utils/roles/meta_lookup.py beside get_role_run_after, and the generator joins it onto the role id as application_deploy_ids for the three places the Quick Setup block spells the id out. A role without companions renders byte-identical, which the README lint confirms across every role.
This declares where e74a585402 derived: a role asking for networks.internet.dns names it in its own tasks, so the generator can read it. A companion cannot be read that way. The broker's tasks state only that the model must resolve, and deploying a backend is one of two remedies, the other being a provider key on svc-ai-litellm; which one the documented path takes is a decision rather than a derivation, so it is declared and the fail_msg names both.
Declaring services.ollama on the broker instead was tried and reverted: it pulls svc-ai-ollama into the closure, which a probe confirms, but the literal include filter leaves it out of group_names all the same, so the model server would run while the assert still fired.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
290 lines
10 KiB
Python
290 lines
10 KiB
Python
import ast
|
|
import re
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import ClassVar
|
|
|
|
from jinja2 import Environment, StrictUndefined, select_autoescape
|
|
|
|
from utils.cache.files import PROJECT_ROOT
|
|
from utils.cache.yaml import load_yaml
|
|
|
|
_AI_VARS = Path(PROJECT_ROOT) / "group_vars" / "all" / "16_ai.yml"
|
|
|
|
|
|
def _ansible_bool(value):
|
|
if isinstance(value, str):
|
|
return value.strip().lower() in ("true", "yes", "on", "1")
|
|
return bool(value)
|
|
|
|
|
|
def _stub_lookup(preload_models, lmstudio_models):
|
|
def lookup(kind, role, path, *args):
|
|
if (kind, path) == ("config", "services.ollama.preload_models"):
|
|
return preload_models
|
|
if (kind, path) == ("config", "services.lmstudio.preload_models"):
|
|
return lmstudio_models
|
|
raise AssertionError(f"unexpected lookup({kind!r}, {role!r}, {path!r})")
|
|
|
|
return lookup
|
|
|
|
|
|
class TestLitellmChatModel(unittest.TestCase):
|
|
"""The gateway only publishes a model it has a backend for
|
|
(roles/svc-ai-litellm/templates/config.yaml.j2). These expressions decide
|
|
what consumers are told to ask for, so naming an unpublished model makes
|
|
every prompt fail with a 400.
|
|
"""
|
|
|
|
env: ClassVar[Environment]
|
|
source: ClassVar[dict]
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.env = Environment(undefined=StrictUndefined, autoescape=select_autoescape())
|
|
cls.env.tests["search"] = lambda value, pattern: bool(
|
|
re.search(pattern, str(value))
|
|
)
|
|
cls.env.filters["bool"] = _ansible_bool
|
|
cls.source = load_yaml(_AI_VARS)
|
|
|
|
def _render(
|
|
self,
|
|
name,
|
|
*,
|
|
roles,
|
|
api_key,
|
|
preload_models=(),
|
|
lmstudio_models=(),
|
|
remote_aliases=None,
|
|
):
|
|
return (
|
|
self.env.from_string(self.source[name])
|
|
.render(
|
|
LITELLM_BACKEND_ROLES=list(roles),
|
|
LITELLM_OLLAMA_BACKEND=str("svc-ai-ollama" in roles),
|
|
LITELLM_LMSTUDIO_BACKEND=str("svc-ai-lmstudio" in roles),
|
|
AI_REMOTE_ALIASES=(
|
|
list(remote_aliases)
|
|
if remote_aliases is not None
|
|
else (["openrouter/auto"] if api_key else [])
|
|
),
|
|
lookup=_stub_lookup(
|
|
[{"alias": alias, "name": alias} for alias in preload_models],
|
|
[
|
|
{"alias": alias, "name": f"{alias}-gguf"}
|
|
for alias in lmstudio_models
|
|
],
|
|
),
|
|
)
|
|
.strip()
|
|
)
|
|
|
|
def _both(self, **kwargs):
|
|
return (
|
|
self._render("LITELLM_CHAT_MODEL", **kwargs),
|
|
self._render("LITELLM_CHAT_MODEL_SERVED", **kwargs),
|
|
)
|
|
|
|
def test_no_backend_and_no_key_names_no_model(self):
|
|
model, served = self._both(roles=["web-app-mattermost"], api_key="")
|
|
self.assertEqual(model, "")
|
|
self.assertEqual(served, "False")
|
|
|
|
def test_no_backend_with_a_key_uses_openrouter(self):
|
|
model, served = self._both(roles=["web-app-mattermost"], api_key="sk-test")
|
|
self.assertEqual(model, "openrouter/auto")
|
|
self.assertEqual(served, "True")
|
|
|
|
def test_a_declared_mock_answers_when_no_backend_preloaded_anything(self):
|
|
model = self._render(
|
|
"LITELLM_CHAT_MODEL",
|
|
roles=["svc-ai-ollama"],
|
|
api_key="",
|
|
preload_models=[],
|
|
remote_aliases=["mock/deterministic"],
|
|
)
|
|
self.assertEqual(
|
|
model,
|
|
"mock/deterministic",
|
|
"a CI round that deploys the backend without preloading must fall to the mock",
|
|
)
|
|
|
|
def test_a_preloaded_model_wins_over_the_mock(self):
|
|
model = self._render(
|
|
"LITELLM_CHAT_MODEL",
|
|
roles=["svc-ai-ollama"],
|
|
api_key="",
|
|
preload_models=["qwen2.5:0.5b"],
|
|
remote_aliases=["mock/deterministic"],
|
|
)
|
|
self.assertEqual(
|
|
model,
|
|
"qwen2.5:0.5b",
|
|
"the round that loads a real model must prove that model, not the mock",
|
|
)
|
|
|
|
def test_without_a_mock_the_chain_is_unchanged(self):
|
|
self.assertEqual(
|
|
self._render(
|
|
"LITELLM_CHAT_MODEL", roles=["web-app-mattermost"], api_key="sk-test"
|
|
),
|
|
"openrouter/auto",
|
|
"production declares no mock, so it must resolve exactly as before",
|
|
)
|
|
|
|
def test_lmstudio_wins_over_the_openrouter_fallback(self):
|
|
model, served = self._both(
|
|
roles=["svc-ai-lmstudio"], api_key="", lmstudio_models=["qwen2.5:0.5b"]
|
|
)
|
|
self.assertEqual(model, "qwen2.5:0.5b")
|
|
self.assertEqual(served, "True")
|
|
|
|
def test_lmstudio_without_a_preloaded_model_serves_nothing(self):
|
|
served = self._render(
|
|
"LITELLM_CHAT_MODEL_SERVED", roles=["svc-ai-lmstudio"], api_key=""
|
|
)
|
|
self.assertEqual(served, "False")
|
|
|
|
def test_both_backends_name_the_same_model(self):
|
|
alias = "qwen2.5:0.5b"
|
|
with_ollama = self._render(
|
|
"LITELLM_CHAT_MODEL",
|
|
roles=["svc-ai-ollama"],
|
|
api_key="",
|
|
preload_models=[alias],
|
|
)
|
|
with_lmstudio = self._render(
|
|
"LITELLM_CHAT_MODEL",
|
|
roles=["svc-ai-lmstudio"],
|
|
api_key="",
|
|
lmstudio_models=[alias],
|
|
)
|
|
self.assertEqual(with_ollama, with_lmstudio)
|
|
self.assertEqual(with_ollama, alias)
|
|
|
|
def test_ollama_serves_its_first_non_embedding_model(self):
|
|
model, served = self._both(
|
|
roles=["svc-ai-ollama"],
|
|
api_key="",
|
|
preload_models=["nomic-embed-text", "llama3.2"],
|
|
)
|
|
self.assertEqual(model, "llama3.2")
|
|
self.assertEqual(served, "True")
|
|
|
|
def test_ollama_with_only_embedding_models_serves_nothing(self):
|
|
served = self._render(
|
|
"LITELLM_CHAT_MODEL_SERVED",
|
|
roles=["svc-ai-ollama"],
|
|
api_key="",
|
|
preload_models=["nomic-embed-text"],
|
|
)
|
|
self.assertEqual(served, "False")
|
|
|
|
def test_the_model_is_named_exactly_when_one_is_served(self):
|
|
cases = (
|
|
(["web-app-mattermost"], "", (), ()),
|
|
(["web-app-mattermost"], "sk-test", (), ()),
|
|
(["svc-ai-lmstudio"], "", (), ()),
|
|
(["svc-ai-lmstudio"], "", (), ("qwen2.5:0.5b",)),
|
|
(["svc-ai-ollama"], "", ("llama3.2",), ()),
|
|
)
|
|
for roles, api_key, preload, lmstudio in cases:
|
|
with self.subTest(roles=roles, api_key=bool(api_key), lmstudio=lmstudio):
|
|
model, served = self._both(
|
|
roles=roles,
|
|
api_key=api_key,
|
|
preload_models=preload,
|
|
lmstudio_models=lmstudio,
|
|
)
|
|
self.assertEqual(bool(model), served == "True")
|
|
|
|
|
|
class TestAiRemoteAliases(unittest.TestCase):
|
|
models: ClassVar[list]
|
|
template: ClassVar[str]
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.models = [
|
|
{"alias": "openrouter/auto", "provider": "openrouter"},
|
|
{"alias": "openrouter/ternary-bonsai-2-27b", "provider": "openrouter"},
|
|
{"alias": "openai/gpt-4o-mini", "provider": "openai"},
|
|
]
|
|
cls.template = load_yaml(_AI_VARS)["AI_REMOTE_ALIASES"]
|
|
|
|
def _render(self, enabled):
|
|
models = self.models
|
|
|
|
def lookup(kind, *terms):
|
|
if kind == "config":
|
|
self.assertEqual(
|
|
terms, ("svc-ai-litellm", "services.litellm.remote_models")
|
|
)
|
|
return models
|
|
if kind == "api_enabled":
|
|
return terms[0] in enabled
|
|
raise AssertionError(f"unexpected lookup({kind!r}, {terms!r})")
|
|
|
|
env = Environment(undefined=StrictUndefined, autoescape=False) # noqa: S701 - renders a Python list literal, not markup
|
|
return ast.literal_eval(
|
|
env.from_string(self.template).render(
|
|
lookup=lookup, AI_MOCK_PROVIDER="mock"
|
|
)
|
|
)
|
|
|
|
def test_every_model_of_an_enabled_provider_is_listed(self):
|
|
self.assertEqual(
|
|
self._render({"openrouter"}),
|
|
["openrouter/auto", "openrouter/ternary-bonsai-2-27b"],
|
|
)
|
|
|
|
def test_no_enabled_provider_lists_nothing(self):
|
|
self.assertEqual(self._render(set()), [])
|
|
|
|
|
|
class TestAiAgentModel(unittest.TestCase):
|
|
"""Every agent the broker starts asks the gateway for this alias, and
|
|
roles/svc-ai-agent-broker/tasks/00_core.yml stops the deploy when it is
|
|
empty. A deployment that carries neither a mock nor a backend nor a
|
|
provider key resolves to '', so the two sources are pinned together.
|
|
"""
|
|
|
|
template: ClassVar[str]
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.template = load_yaml(_AI_VARS)["AI_AGENT_MODEL"]
|
|
|
|
def _render(self, *, mock_aliases, chat_model):
|
|
env = Environment(undefined=StrictUndefined, autoescape=False) # noqa: S701 - renders a bare alias, not markup
|
|
return env.from_string(self.template).render(
|
|
AI_MOCK_ALIASES=list(mock_aliases), LITELLM_CHAT_MODEL=chat_model
|
|
)
|
|
|
|
def test_a_declared_mock_is_preferred_over_the_served_model(self):
|
|
self.assertEqual(
|
|
self._render(
|
|
mock_aliases=["mock/deterministic"], chat_model="qwen2.5:0.5b"
|
|
),
|
|
"mock/deterministic",
|
|
)
|
|
|
|
def test_without_a_mock_the_served_model_answers(self):
|
|
self.assertEqual(
|
|
self._render(mock_aliases=[], chat_model="qwen2.5:0.5b"), "qwen2.5:0.5b"
|
|
)
|
|
|
|
def test_neither_a_mock_nor_a_served_model_names_nothing(self):
|
|
self.assertEqual(
|
|
self._render(mock_aliases=[], chat_model=""),
|
|
"",
|
|
"a broker deployed without a backend role and without a provider "
|
|
"key has no model to start its agents on, which 00_core.yml turns "
|
|
"into a stopped deploy rather than agents that all answer 400",
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|