mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 19:01:54 +00:00
kevinveenbirkenbachandClaude Opus 5
c7507150e0
Four tasks in this file talk to the gateway, and each carries its own
tolerance for it not being up: 60 seconds for the keys, 50 for the closure
proof, 50 for the model warm-up. The first one to run is the keys, so a gateway
that takes longer than a minute fails there and the later budgets never apply.
54a6f3b533 already named this when it added the warm-up: "Five retries cover a
gateway that is not listening yet." It gave the last task 50 seconds and left
the first one at the 60 it already had.
One wait in front of all four, on the signal the failure actually reports.
CI shows ConnectionRefusedError from urllib after eleven attempts, which is the
TCP layer, so the probe opens a socket rather than asking an HTTP endpoint whose
path would be a guess. python3 is what the failing task itself runs in that
container, so nothing new has to exist there.
A healthcheck would have been the obvious shape and would have changed nothing:
sys-svc-compose/tasks/utils/post_deploy.yml waits on the database container's
health in compose and on swarm tasks being Running in swarm, and reads the
application container's health in neither.
This does not explain every job in that cluster. One role's rescue artifact
shows all three replicas reaching "Proxy initialized with Config" while
postgres resets and then terminates their connections (SqlState 57P01), which
is a different failure wearing the same stack trace. Where that is the cause,
this makes the deploy fail at a task that names the gateway instead of at the
one provisioning keys.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
52 lines
3.0 KiB
YAML
52 lines
3.0 KiB
YAML
---
|
|
- name: "🔍 Resolve {{ application_id }} hosting node + container id"
|
|
ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml') }}"
|
|
vars:
|
|
resolve_service_key: "{{ entity_name }}"
|
|
resolve_output_prefix: LITELLM
|
|
resolve_compose_cid: "{{ LITELLM_CONTAINER }}"
|
|
|
|
- name: "⏳ Wait until the gateway accepts connections on its own port"
|
|
ansible.builtin.shell: | # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node
|
|
container exec -i {{ LITELLM_LOCAL_CID | quote }} python3 -c 'import socket; socket.create_connection(("127.0.0.1", {{ LITELLM_PORT }}), 5).close()'
|
|
register: litellm_gateway_listening
|
|
retries: 60
|
|
delay: 5
|
|
until: litellm_gateway_listening.rc == 0
|
|
changed_when: false
|
|
delegate_to: "{{ LITELLM_HOST_NODE }}"
|
|
|
|
- name: "🔑 Ensure a virtual key exists per consumer"
|
|
ansible.builtin.shell: | # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node
|
|
container exec -i -e LITELLM_KEYS_PAYLOAD={{ LITELLM_CONSUMER_KEYS | to_json | quote }} -e LITELLM_MK={{ LITELLM_MASTER_KEY | quote }} -e LITELLM_PORT={{ LITELLM_PORT | quote }} {{ LITELLM_LOCAL_CID | quote }} python3 -c {{ lookup('file', lookup('path_absolute', 'roles/svc-ai-litellm/files/python/ensure_keys.py')) | quote }}
|
|
register: litellm_keys_result
|
|
retries: 10
|
|
delay: 6
|
|
until: litellm_keys_result.rc == 0
|
|
changed_when: "'CHANGED' in litellm_keys_result.stdout"
|
|
no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
|
|
delegate_to: "{{ LITELLM_HOST_NODE }}"
|
|
|
|
- name: "🛡️ Prove the gateway is closed and revocation stays per consumer"
|
|
ansible.builtin.shell: | # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node
|
|
container exec -i -e LITELLM_MK={{ LITELLM_MASTER_KEY | quote }} -e LITELLM_PORT={{ LITELLM_PORT | quote }} -e LITELLM_KEY={{ (LITELLM_CONSUMER_KEYS | first).key | quote }} {{ LITELLM_LOCAL_CID | quote }} python3 -c {{ lookup('file', lookup('path_absolute', 'roles/svc-ai-litellm/files/python/probe_auth.py')) | quote }}
|
|
register: litellm_auth_probe
|
|
retries: 5
|
|
delay: 10
|
|
until: litellm_auth_probe.rc == 0
|
|
changed_when: false
|
|
no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
|
|
delegate_to: "{{ LITELLM_HOST_NODE }}"
|
|
|
|
- name: "🔥 Warm the chat model so the first prompt is not a cold load"
|
|
ansible.builtin.shell: | # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node
|
|
container exec -i -e LITELLM_MK={{ LITELLM_MASTER_KEY | quote }} -e LITELLM_PORT={{ LITELLM_PORT | quote }} -e LITELLM_MODEL={{ LITELLM_CHAT_MODEL | quote }} {{ LITELLM_LOCAL_CID | quote }} python3 -c {{ lookup('file', lookup('path_absolute', 'roles/svc-ai-litellm/files/python/warm_model.py')) | quote }}
|
|
register: litellm_warm_result
|
|
retries: 5
|
|
delay: 10
|
|
until: litellm_warm_result.rc == 0
|
|
changed_when: false
|
|
no_log: "{{ MASK_CREDENTIALS_IN_LOGS | bool }}"
|
|
delegate_to: "{{ LITELLM_HOST_NODE }}"
|
|
when: LITELLM_CHAT_MODEL_SERVED | bool
|